Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Cache impersonated native query validation (#85842)

Alexander Polyankin40981248c541
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 905c656cadf..379b5a5bf10 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -4242,8 +4242,9 @@
enterprise/permissions.impersonation
{:team "Security Engineering"
:ns-prefix "metabase-enterprise.impersonation"
- :api #{metabase-enterprise.impersonation.api}
+ :api #{metabase-enterprise.impersonation.api metabase-enterprise.impersonation.init}
:uses #{api
+ app-db
audit-app
driver
lib
@@ -4252,7 +4253,10 @@
models
premium-features
query-processor
+ settings
+ task
util
+ util.config
warehouses.schema
enterprise/permissions.sandbox}
:model-exports #{:model/ConnectionImpersonation}
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index 8cc8395e20e..52a4f3b9606 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -19,6 +19,7 @@
[metabase-enterprise.database-replication.init]
[metabase-enterprise.dependencies.init]
[metabase-enterprise.gsheets.init]
+ [metabase-enterprise.impersonation.init]
[metabase-enterprise.mcp.init]
[metabase-enterprise.memoize-monitor.init]
[metabase-enterprise.metabot.init]
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/db.clj b/enterprise/backend/src/metabase_enterprise/impersonation/db.clj
index 4dbd0765ddb..e9c1311e5b8 100644
--- a/enterprise/backend/src/metabase_enterprise/impersonation/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/db.clj
@@ -3,7 +3,9 @@
additional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."
(:require
[malli.util :as mut]
+ [metabase-enterprise.impersonation.models.impersonation-query-cache :as impersonation-query-cache]
[metabase-enterprise.impersonation.schema :as impersonation.schema]
+ [metabase.app-db.core :as app-db]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -87,3 +89,35 @@
current user, such as an internal or unauthenticated context)."
[user-id :- [:maybe ::lib.schema.id/user]]
(t2/select-fn-set :group_id :model/PermissionsGroupMembership :user_id user-id))
+
+(mu/defn query-cache :- [:maybe ::impersonation.schema/impersonation-query-cache]
+ "The ImpersonationQueryCache with the lookup key of `input`, or nil."
+ [{:keys [input_query input_engine input_allow_write input_admin input_version]}
+ :- ::impersonation.schema/impersonation-query-cache.update]
+ (t2/select-one :model/ImpersonationQueryCache
+ :input_query_hash (impersonation-query-cache/query-hash input_query)
+ :input_engine input_engine
+ :input_allow_write input_allow_write
+ :input_admin input_admin
+ :input_version input_version))
+
+(mu/defn update-or-insert-query-cache! :- ms/PositiveInt
+ "Update the ImpersonationQueryCache with the lookup key of `row` to `row`, or insert `row` when there is none."
+ [{:keys [input_query input_engine input_allow_write input_admin input_version] :as row}
+ :- ::impersonation.schema/impersonation-query-cache.update]
+ (app-db/update-or-insert! :model/ImpersonationQueryCache
+ {:input_query_hash (impersonation-query-cache/query-hash input_query)
+ :input_engine input_engine
+ :input_allow_write input_allow_write
+ :input_admin input_admin
+ :input_version input_version}
+ (constantly row)))
+
+(mu/defn delete-outdated-query-caches! :- :int
+ "Delete the ImpersonationQueryCaches cached by a Metabase version other than `version` or before `cutoff`,
+ returning the number deleted."
+ [version :- :string
+ cutoff :- (ms/InstanceOfClass java.time.OffsetDateTime)]
+ (t2/delete! :model/ImpersonationQueryCache {:where [:or
+ [:not= :input_version version]
+ [:< :created_at cutoff]]}))
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/init.clj b/enterprise/backend/src/metabase_enterprise/impersonation/init.clj
new file mode 100644
index 00000000000..f22af04111f
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/init.clj
@@ -0,0 +1,4 @@
+(ns metabase-enterprise.impersonation.init
+ (:require
+ [metabase-enterprise.impersonation.settings]
+ [metabase-enterprise.impersonation.task.query-cache-cleanup]))
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj
index 90315e0191e..6959ab92930 100644
--- a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj
@@ -1,17 +1,21 @@
(ns metabase-enterprise.impersonation.middleware
(:require
[metabase-enterprise.impersonation.driver :as impersonation.driver]
+ [metabase-enterprise.impersonation.query-cache :as query-cache]
[metabase.api.common :as api]
[metabase.driver :as driver]
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema :as lib.schema]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.interface :as qp.i]
;; legacy usage -- don't do things like this going forward
^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
- [metabase.util.i18n :refer [tru]]))
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.malli :as mu]))
(defn- validates-native-queries?
"Does `driver` supply its own `driver/validate-impersonated-query`?"
@@ -28,14 +32,35 @@
(throw (ex-info (tru "Connection impersonation does not support native queries on this database.")
{:type qp.error-type/invalid-query}))))
+(mu/defn- validate-native-stage :- ::lib.schema/stage.native
+ "Validate the native `stage` of `query` for `driver` on its own, through the impersonation query cache."
+ [driver :- :keyword
+ query :- ::lib.schema/query
+ stage :- ::lib.schema/stage.native]
+ (let [admin? api/*is-superuser?*]
+ (assoc stage :native
+ (query-cache/validated-query
+ driver
+ driver.settings/*impersonation-allow-write?*
+ admin?
+ (:native stage)
+ #(-> (driver/validate-impersonated-query
+ driver
+ (cond-> (assoc query :stages [stage]) admin? (assoc :impersonation/admin? true)))
+ :stages
+ first
+ :native)))))
+
(defn- validate-impersonated-query
- "Enforce the single-statement rule on `query` for `driver`."
+ "Enforce the single-statement rule on every native stage of `query` for `driver`, including stages nested in joins."
[driver database query]
(if (driver.u/supports? driver :connection-impersonation database)
(do (assert-native-validation-supported driver query)
- (driver/validate-impersonated-query
- driver
- (cond-> query api/*is-superuser?* (assoc :impersonation/admin? true))))
+ (lib/walk-stages
+ query
+ (fn [_query _path stage]
+ (when (lib/native-stage? stage)
+ (validate-native-stage driver query stage)))))
query))
(defenterprise apply-impersonation
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.clj b/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.clj
new file mode 100644
index 00000000000..2350a16130d
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.clj
@@ -0,0 +1,35 @@
+(ns metabase-enterprise.impersonation.models.impersonation-query-cache
+ "Model definition for ImpersonationQueryCaches, the cached validation results of native query stages run on
+ impersonated databases."
+ (:require
+ [buddy.core.codecs :as codecs]
+ [buddy.core.hash :as buddy-hash]
+ [metabase.models.interface :as mi]
+ [metabase.util.malli :as mu]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/ImpersonationQueryCache [_model] :impersonation_query_cache)
+
+(doto :model/ImpersonationQueryCache
+ (derive :metabase/model)
+ (derive :hook/created-at-timestamped?))
+
+(t2/deftransforms :model/ImpersonationQueryCache
+ {:input_engine mi/transform-keyword
+ :result mi/transform-keyword})
+
+(mu/defn query-hash :- :string
+ "The SHA-256 hex of the native SQL `query`, which looks an ImpersonationQueryCache up."
+ [query :- :string]
+ (codecs/bytes->hex (buddy-hash/sha256 ^String query)))
+
+(t2/define-before-insert :model/ImpersonationQueryCache
+ [row]
+ (assoc row :input_query_hash (query-hash (:input_query row))))
+
+(t2/define-before-update :model/ImpersonationQueryCache
+ [row]
+ (let [changes (t2/changes row)]
+ (cond-> row
+ (contains? changes :input_query) (assoc :input_query_hash (query-hash (:input_query changes))))))
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.clj b/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.clj
new file mode 100644
index 00000000000..31cccb97ded
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.clj
@@ -0,0 +1,82 @@
+(ns metabase-enterprise.impersonation.query-cache
+ "Caches the result of validating each native stage of an impersonated query in the app DB, since validation parses
+ the SQL and can take seconds."
+ (:require
+ [metabase-enterprise.impersonation.db :as impersonation.db]
+ [metabase-enterprise.impersonation.schema :as impersonation.schema]
+ [metabase.config.core :as config]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu])
+ (:import
+ (clojure.lang ExceptionInfo)))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn- query-cache-input :- ::impersonation.schema/impersonation-query-cache.update
+ "The lookup key of validating the native SQL `native` for `engine`."
+ [engine :- :keyword
+ allow-write? :- [:maybe :boolean]
+ admin? :- [:maybe :boolean]
+ native :- :string]
+ {:input_version config/mb-version-string
+ :input_engine engine
+ :input_allow_write (boolean allow-write?)
+ :input_admin (boolean admin?)
+ :input_query native})
+
+(mu/defn- cached-result :- [:maybe ::impersonation.schema/impersonation-query-cache]
+ "The cached result for `input`, or nil when there is none or the cache cannot be read."
+ [input :- ::impersonation.schema/impersonation-query-cache.update]
+ (try
+ (let [row (impersonation.db/query-cache input)]
+ (when (= (:input_query input) (:input_query row))
+ row))
+ (catch Throwable e
+ (log/errorf "Error reading the impersonation query cache: %s" (ex-message e))
+ nil)))
+
+(mu/defn- save-result! :- :nil
+ "Cache `result` for `input`, replacing a row with the same lookup key, or log why the cache cannot be written."
+ [input :- ::impersonation.schema/impersonation-query-cache.update
+ result :- ::impersonation.schema/impersonation-query-cache.update]
+ (try
+ (impersonation.db/update-or-insert-query-cache! (merge input result))
+ (catch Throwable e
+ (log/errorf "Error writing the impersonation query cache: %s" (ex-message e))))
+ nil)
+
+(mu/defn- replay-result :- :string
+ "The validated SQL of the cached `result` for the native SQL `native`, or the refusal it records thrown."
+ [{:keys [result result_query result_reason]} :- ::impersonation.schema/impersonation-query-cache
+ native :- :string
+ allow-write? :- [:maybe :boolean]]
+ (if (= result :valid)
+ result_query
+ (throw (driver.sql/impersonated-query-refusal result native allow-write? result_reason))))
+
+(mu/defn- validate-and-save! :- :string
+ "Call `validate` and cache what it returns, or the refusal it throws when that refusal records its result."
+ [input :- ::impersonation.schema/impersonation-query-cache.update
+ validate :- fn?]
+ (try
+ (let [validated (validate)]
+ (save-result! input {:result :valid, :result_query validated})
+ validated)
+ (catch ExceptionInfo e
+ (when-let [result (#{:unparseable :unvalidatable :invalid} (:impersonation/result (ex-data e)))]
+ (save-result! input {:result result, :result_reason (:reason (ex-data e))}))
+ (throw e))))
+
+(mu/defn validated-query :- :string
+ "The validated SQL to run in place of the native stage SQL `native` on `engine`, from the cache or else by calling
+ `validate`, which returns that SQL or throws a refusal."
+ [engine :- :keyword
+ allow-write? :- [:maybe :boolean]
+ admin? :- [:maybe :boolean]
+ native :- :string
+ validate :- fn?]
+ (let [input (query-cache-input engine allow-write? admin? native)]
+ (if-let [cached (cached-result input)]
+ (replay-result cached native allow-write?)
+ (validate-and-save! input validate))))
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj b/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj
index 6c2f8203855..d06ee8d4605 100644
--- a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj
@@ -18,3 +18,28 @@
[:db_id {:optional true} [:maybe ::lib.schema.id/database]]
[:group_id {:optional true} [:maybe ms/PositiveInt]]
[:attribute {:optional true} [:maybe :string]]])
+
+(mr/def ::impersonation-query-cache.result
+ "The outcome of validating an impersonated native query, as an ImpersonationQueryCache records it."
+ [:enum :valid :unparseable :unvalidatable :invalid])
+
+(mr/def ::impersonation-query-cache
+ "An ImpersonationQueryCache as selected from the app DB: every column of `:impersonation_query_cache`."
+ [:merge
+ ::impersonation-query-cache.update
+ [:map {:closed true}
+ [:id ms/PositiveInt]
+ [:input_query_hash :string]]])
+
+(mr/def ::impersonation-query-cache.update
+ "What an update (or insert) of an ImpersonationQueryCache accepts: every column of `:impersonation_query_cache` except `id`, all optional."
+ [:map {:closed true}
+ [:input_version {:optional true} :string]
+ [:input_engine {:optional true} :keyword]
+ [:input_allow_write {:optional true} :boolean]
+ [:input_admin {:optional true} :boolean]
+ [:input_query {:optional true} :string]
+ [:result {:optional true} ::impersonation-query-cache.result]
+ [:result_query {:optional true} [:maybe :string]]
+ [:result_reason {:optional true} [:maybe :string]]
+ [:created_at {:optional true} (ms/InstanceOfClass java.time.temporal.Temporal)]])
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/settings.clj b/enterprise/backend/src/metabase_enterprise/impersonation/settings.clj
new file mode 100644
index 00000000000..672a1df97b4
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/settings.clj
@@ -0,0 +1,13 @@
+(ns metabase-enterprise.impersonation.settings
+ (:require
+ [metabase.settings.core :refer [defsetting]]
+ [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting impersonation-query-cache-retention-days
+ (deferred-tru "Cached validation results of impersonated native queries are deleted once they are this many days old.")
+ :encryption :no
+ :visibility :internal
+ :default 30
+ :type :positive-integer
+ :export? false
+ :doc false)
diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.clj b/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.clj
new file mode 100644
index 00000000000..b69759ba879
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.clj
@@ -0,0 +1,45 @@
+(ns metabase-enterprise.impersonation.task.query-cache-cleanup
+ "Scheduled task that deletes impersonation query cache rows cached by another Metabase version or more than
+ [[impersonation.settings/impersonation-query-cache-retention-days]] ago."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase-enterprise.impersonation.db :as impersonation.db]
+ [metabase-enterprise.impersonation.settings :as impersonation.settings]
+ [metabase.config.core :as config]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private cleanup-job-key (jobs/key "metabase.task.impersonation.query-cache-cleanup.job"))
+(def ^:private cleanup-trigger-key (triggers/key "metabase.task.impersonation.query-cache-cleanup.trigger"))
+
+(mu/defn- cleanup-query-cache! :- :nil
+ "Delete the outdated impersonation query cache rows."
+ []
+ (let [cutoff (t/minus (t/offset-date-time) (t/days (impersonation.settings/impersonation-query-cache-retention-days)))
+ deleted (impersonation.db/delete-outdated-query-caches! config/mb-version-string cutoff)]
+ (log/infof "Impersonation query cache cleanup complete. Deleted %d row(s)." deleted)))
+
+(task/defjob ImpersonationQueryCacheCleanup
+ "Delete outdated impersonation query cache rows"
+ {:saved-class "metabase_enterprise.impersonation.task.query_cache_cleanup.ImpersonationQueryCacheCleanup"
+ :concurrent? false}
+ [_ctx]
+ (cleanup-query-cache!))
+
+(defmethod task/init! ::ImpersonationQueryCacheCleanup
+ [_]
+ (let [job (jobs/build
+ (jobs/of-type ImpersonationQueryCacheCleanup)
+ (jobs/with-identity cleanup-job-key))
+ trigger (triggers/build
+ (triggers/with-identity cleanup-trigger-key)
+ (triggers/start-now)
+ (triggers/with-schedule
+ (cron/cron-schedule "37 41 23 * * ?")))]
+ (task/schedule-task! job trigger)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index dea7f19b834..d008da9317b 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -111,6 +111,7 @@
"ExplorationThread"
"ExplorationThreadTimeline"
"ImplicitAction"
+ "ImpersonationQueryCache"
"LoginHistory"
"McpFeedback"
"McpGroupPermission"
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
index de84b678d69..9bdce39d5ff 100644
--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
@@ -7,6 +7,7 @@
[clojure.test :refer :all]
[metabase-enterprise.impersonation.driver :as impersonation.driver]
[metabase-enterprise.impersonation.middleware :as impersonation.middleware]
+ [metabase-enterprise.impersonation.query-cache :as query-cache]
[metabase-enterprise.impersonation.util-test :as impersonation.util-test]
[metabase-enterprise.test :as met]
[metabase.actions.execution :as actions.execution]
@@ -36,6 +37,7 @@
[toucan2.core :as t2]
[toucan2.tools.with-temp :as t2.with-temp])
(:import
+ (clojure.lang ExceptionInfo)
(com.mchange.v2.c3p0 PooledDataSource)
(java.sql Connection)
(java.util.concurrent CountDownLatch)))
@@ -1181,6 +1183,32 @@
"SELECT set_config('role', 'none', false); DROP TABLE table"
"DO $$ BEGIN EXECUTE 'SET ROLE NONE; DROP TABLE table'; END $$;"))))))))))
+(deftest cached-refusal-fails-query-like-uncached-refusal-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :connection-impersonation)
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (impersonation.util-test/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]
+ :attributes {"impersonation_attr" "impersonation_role"}}
+ (let [run-query (fn [sql]
+ (try
+ (qp/process-query (lib/native-query (mt/metadata-provider) sql))
+ (catch ExceptionInfo e
+ {:message (ex-message e)
+ :refusal-message (ex-message (ex-cause e))
+ :refusal-data (ex-data (ex-cause e))})))]
+ (doseq [[result sql] [[:unparseable "SELECT ("]
+ [:invalid "SELECT 1; SELECT 2"]
+ [:unvalidatable (str "SELECT 1 WHERE 1 IN (" (str/join ", " (range 3000)) ")")]]]
+ (testing result
+ (let [sql (str "/* " (mt/random-name) " */ " sql)
+ uncached (mt/with-dynamic-fn-redefs [query-cache/validated-query (fn [_engine _allow-write? _admin? _native validate]
+ (validate))]
+ (run-query sql))]
+ (is (= result (get-in uncached [:refusal-data :impersonation/result])))
+ (is (= uncached (run-query sql)))
+ (is (t2/exists? :model/ImpersonationQueryCache :input_query sql))
+ (is (= uncached (run-query sql))))))))))))
+
(deftest reject-non-select-statements-for-admin-on-impersonated-db-test
(testing "Admin users are also validated on impersonated databases"
(mt/test-drivers (mt/normal-drivers-with-feature :connection-impersonation)
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.clj
new file mode 100644
index 00000000000..93b0724732c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.clj
@@ -0,0 +1,106 @@
+(ns metabase-enterprise.impersonation.query-cache-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.impersonation.models.impersonation-query-cache :as impersonation-query-cache]
+ [metabase-enterprise.impersonation.query-cache :as query-cache]
+ [metabase-enterprise.impersonation.task.query-cache-cleanup :as query-cache-cleanup]
+ [metabase.config.core :as config]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (clojure.lang ExceptionInfo)))
+
+(deftest valid-result-is-cached-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)
+ calls (atom 0)
+ validate (fn [] (swap! calls inc) "SELECT 1")]
+ (is (= "SELECT 1" (query-cache/validated-query :postgres false false native validate)))
+ (is (= "SELECT 1" (query-cache/validated-query :postgres false false native validate)))
+ (is (= 1 @calls)))))
+
+(deftest refusal-with-result-is-cached-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (doseq [[result allow-write? reason message] [[:unparseable false nil #"Unable to parse native query"]
+ [:unvalidatable false "large-literal-list" #"Unable to validate this native query"]
+ [:invalid false nil #"Must be a single select statement"]
+ [:invalid true nil #"Must be a single insert, update, or delete statement"]]]
+ (testing (str result " allow-write? " allow-write?)
+ (let [native (mt/random-name)
+ refusal (driver.sql/impersonated-query-refusal result native allow-write? reason)]
+ (is (thrown-with-msg? ExceptionInfo message
+ (query-cache/validated-query :postgres allow-write? false native #(throw refusal))))
+ (is (thrown-with-msg? ExceptionInfo message
+ (query-cache/validated-query :postgres allow-write? false native (constantly "SELECT 1"))))
+ (is (= (ex-data refusal)
+ (try
+ (query-cache/validated-query :postgres allow-write? false native (constantly "SELECT 1"))
+ (catch ExceptionInfo e
+ (ex-data e))))))))))
+
+(deftest refusal-without-result-is-not-cached-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)
+ calls (atom 0)
+ validate (fn [] (swap! calls inc) (throw (ex-info "Boom" {})))]
+ (is (thrown-with-msg? ExceptionInfo #"Boom" (query-cache/validated-query :postgres false false native validate)))
+ (is (thrown-with-msg? ExceptionInfo #"Boom" (query-cache/validated-query :postgres false false native validate)))
+ (is (= 2 @calls)))))
+
+(deftest every-input-is-part-of-key-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)]
+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))
+ (is (= "SELECT 2" (query-cache/validated-query :mysql false false native (constantly "SELECT 2"))))
+ (is (= "SELECT 2" (query-cache/validated-query :postgres true false native (constantly "SELECT 2"))))
+ (is (= "SELECT 2" (query-cache/validated-query :postgres false true native (constantly "SELECT 2"))))
+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false (str native " ") (constantly "SELECT 2")))))))
+
+(deftest other-version-is-not-read-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)]
+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))
+ (t2/update! :model/ImpersonationQueryCache :input_query native {:input_version "v0.0.0 (other)"})
+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false native (constantly "SELECT 2")))))))
+
+(deftest hash-collision-is-replaced-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)]
+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))
+ (t2/update! :impersonation_query_cache :input_query native {:input_query "SELECT 'colliding'"})
+ (testing "row whose input_query differs is not used, and replacing it keeps surrounding transaction usable"
+ (t2/with-transaction [_conn]
+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false native (constantly "SELECT 2"))))
+ (is (= "SELECT 2" (t2/select-one-fn :result_query :model/ImpersonationQueryCache :input_query native))))))))
+
+(deftest hash-follows-input-query-test
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [native (mt/random-name)
+ changed (mt/random-name)]
+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))
+ (is (= (impersonation-query-cache/query-hash native)
+ (t2/select-one-fn :input_query_hash :model/ImpersonationQueryCache :input_query native)))
+ (t2/update! :model/ImpersonationQueryCache :input_query native {:input_query changed})
+ (is (= (impersonation-query-cache/query-hash changed)
+ (t2/select-one-fn :input_query_hash :model/ImpersonationQueryCache :input_query changed))))))
+
+(deftest cleanup-deletes-outdated-rows-test
+ (mt/with-temporary-setting-values [impersonation-query-cache-retention-days 7]
+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]
+ (let [current (mt/random-name)
+ recent (mt/random-name)
+ old (mt/random-name)
+ outdated (mt/random-name)]
+ (doseq [native [current recent old outdated]]
+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1")))
+ (t2/update! :model/ImpersonationQueryCache :input_query recent {:created_at (t/minus (t/offset-date-time) (t/days 6))})
+ (t2/update! :model/ImpersonationQueryCache :input_query old {:created_at (t/minus (t/offset-date-time) (t/days 8))})
+ (t2/update! :model/ImpersonationQueryCache :input_query outdated {:input_version "v0.0.0 (other)"})
+ (#'query-cache-cleanup/cleanup-query-cache!)
+ (is (= #{current recent}
+ (t2/select-fn-set :input_query :model/ImpersonationQueryCache
+ :input_query [:in [current recent old outdated]])))
+ (is (= config/mb-version-string
+ (t2/select-one-fn :input_version :model/ImpersonationQueryCache :input_query current)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 0d35cd6f241..cbb870e2e70 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -156,6 +156,7 @@
:model/ViewLog
:model/Sandbox
:model/ConnectionImpersonation
+ :model/ImpersonationQueryCache
:model/SecurityAdvisory
:model/CloudMigration
:model/Comment
diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yaml
index 0d35b65d4e3..31946f301a4 100644
--- a/resources/migrations/058_update_migrations.yaml
+++ b/resources/migrations/058_update_migrations.yaml
@@ -1692,6 +1692,94 @@ databaseChangeLog:
tableName: core_session
columnName: saml_name_id_format
+ - changeSet:
+ id: v58.2026-10-29T00:00:00
+ author: ranquild
+ comment: Add impersonation_query_cache
+ changes:
+ - createTable:
+ tableName: impersonation_query_cache
+ remarks: Cached validation results of native query stages run on impersonated databases
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ primaryKeyName: pk_impersonation_query_cache
+ - column:
+ name: input_version
+ type: varchar(255)
+ remarks: Metabase version that validated the query; rows from other versions are never read
+ constraints:
+ nullable: false
+ - column:
+ name: input_engine
+ type: varchar(254)
+ remarks: Driver that validated the query
+ constraints:
+ nullable: false
+ - column:
+ name: input_allow_write
+ type: ${boolean.type}
+ remarks: Whether the query was validated as a write statement rather than a select
+ constraints:
+ nullable: false
+ - column:
+ name: input_admin
+ type: ${boolean.type}
+ remarks: Whether the query was validated for an admin, who is allowed more than other users
+ constraints:
+ nullable: false
+ - column:
+ name: input_query_hash
+ type: varchar(64)
+ remarks: SHA-256 hex of input_query, used to look the row up
+ constraints:
+ nullable: false
+ - column:
+ name: input_query
+ type: ${text.type}
+ remarks: Native SQL of the stage after parameter substitution
+ constraints:
+ nullable: false
+ - column:
+ name: result
+ type: varchar(32)
+ remarks: 'Validation outcome: valid, unparseable, unvalidatable or invalid'
+ constraints:
+ nullable: false
+ - column:
+ name: result_query
+ type: ${text.type}
+ remarks: Validated SQL to run in place of input_query, set only when result is valid
+ constraints:
+ nullable: true
+ - column:
+ name: result_reason
+ type: varchar(64)
+ remarks: Why the validator could not check the query, when it says
+ constraints:
+ nullable: true
+ - column:
+ name: created_at
+ type: ${timestamp_type}
+ remarks: Timestamp when the result was cached
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v58.2026-10-29T00:00:01
+ author: ranquild
+ comment: Add the lookup key of impersonation_query_cache
+ changes:
+ - addUniqueConstraint:
+ tableName: impersonation_query_cache
+ columnNames: input_query_hash, input_engine, input_allow_write, input_admin, input_version
+ constraintName: uq_impersonation_query_cache_input
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index a8e637741fb..f42c7c7c2af 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -337,10 +337,28 @@
native-query :- :metabase.lib.schema/native-only-query]
(sql-tools/validate-query driver native-query))
+(mu/defn impersonated-query-refusal :- [:fn #(instance? clojure.lang.ExceptionInfo %)]
+ "The [[metabase.query-processor.error-type/invalid-query]] `ex-info` refusing the impersonated native `sql`
+ with `result`: `:unparseable`, `:unvalidatable` (with the validator's `reason`) or `:invalid`. `write?` says
+ whether the query had to be a write statement rather than a select."
+ [result :- [:enum :unparseable :unvalidatable :invalid]
+ sql :- :string
+ write? :- [:maybe :boolean]
+ reason :- [:maybe :string]]
+ (ex-info (case result
+ :unparseable (tru "Unable to parse native query. There might be something wrong with your query.")
+ :unvalidatable (tru "Unable to validate this native query on an impersonated database. Reduce the size of any large VALUES, IN, or ARRAY lists.")
+ :invalid (if write?
+ (tru "Invalid impersonated native query. Must be a single insert, update, or delete statement.")
+ (tru "Invalid impersonated native query. Must be a single select statement.")))
+ (cond-> {:type qp.error-type/invalid-query
+ :impersonation/result result
+ :sql sql}
+ reason (assoc :reason reason))))
+
(defn- validate-impersonated-native-stage
"Validate one native `stage` for `driver`, returning it with `:native` replaced by the reconstructed,
- validated SQL — or throwing a [[metabase.query-processor.error-type/invalid-query]] `ex-info` when it must
- be refused.
+ validated SQL — or throwing an [[impersonated-query-refusal]] when it must be refused.
`admin?` users run as the full connection user and are never actually impersonated."
[driver stage admin?]
@@ -352,9 +370,7 @@
error
(do
(log/warnf "Failed to parse native query: %s" error)
- (throw (ex-info (tru "Unable to parse native query. There might be something wrong with your query.")
- {:type qp.error-type/invalid-query
- :sql (:native stage)})))
+ (throw (impersonated-query-refusal :unparseable (:native stage) write? nil)))
;; The SQL was not checked at all, as opposed to checked and found to be two statements. Admins run as
;; the full connection user and are never actually impersonated, so validating their native SQL
@@ -365,10 +381,7 @@
stage
(do
(log/warnf "Refusing impersonated native query on %s: cannot validate (%s)" driver reason)
- (throw (ex-info (tru "Unable to validate this native query on an impersonated database. Reduce the size of any large VALUES, IN, or ARRAY lists.")
- {:type qp.error-type/invalid-query
- :reason reason
- :sql (:native stage)}))))
+ (throw (impersonated-query-refusal :unvalidatable (:native stage) write? reason))))
;; Admins may also run a single statement that is not of the required type (e.g. `SHOW TIMEZONE`),
;; but never more than one statement.
@@ -376,11 +389,7 @@
(assoc stage :native sql)
:else
- (throw (ex-info (if write?
- (tru "Invalid impersonated native query. Must be a single insert, update, or delete statement.")
- (tru "Invalid impersonated native query. Must be a single select statement."))
- {:type qp.error-type/invalid-query
- :sql (:native stage)})))))
+ (throw (impersonated-query-refusal :invalid (:native stage) write? nil)))))
(defn validate-impersonated-query*
"Validates a native query by parsing it and ensuring that it is a single statement.
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 78000580a6f..61d70329a39 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -69,6 +69,7 @@
:model/Glossary metabase.glossary.models.glossary
:model/Sandbox metabase-enterprise.sandbox.models.sandbox
:model/ImplicitAction metabase.actions.models
+ :model/ImpersonationQueryCache metabase-enterprise.impersonation.models.impersonation-query-cache
:model/LoginHistory metabase.login-history.models.login-history
:model/McpFeedback metabase.mcp.models.mcp-feedback
:model/McpGroupPermission metabase.mcp.models.mcp-group-permission
diff --git a/test/metabase/app_db/quartz_test.clj b/test/metabase/app_db/quartz_test.clj
index 17a89cdb748..f19288bb293 100644
--- a/test/metabase/app_db/quartz_test.clj
+++ b/test/metabase/app_db/quartz_test.clj
@@ -178,6 +178,7 @@
"metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck"
"metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata"
"metabase_enterprise.entity_retrieval.task.sync.OsiAiContextSync"
+ "metabase_enterprise.impersonation.task.query_cache_cleanup.ImpersonationQueryCacheCleanup"
"metabase_enterprise.mcp.task.mcp_usage_trimmer.McpUsageTrimmer"
"metabase_enterprise.metabot.task.ai_usage_trimmer.AiUsageTrimmer"
"metabase_enterprise.remote_sync.task.import.AutoImport"
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index efe7ece5f8a..e790203758e 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -153,6 +153,7 @@
:model/DependencyStatus
:model/EmbeddingSdkReact18Host
:model/ExplorationQueryResult
+ :model/ImpersonationQueryCache
:model/McpQueryHandle
:model/McpSessionLog
:model/McpToolCallLog