The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Cache impersonated native query validation (#85842)
Alexander Polyankin40981248c541
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 905c656cadf..379b5a5bf10 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -4242,8 +4242,9 @@enterprise/permissions.impersonation{:team "Security Engineering":ns-prefix "metabase-enterprise.impersonation"- :api #{metabase-enterprise.impersonation.api}+ :api #{metabase-enterprise.impersonation.api metabase-enterprise.impersonation.init}:uses #{api+ app-dbaudit-appdriverlib@@ -4252,7 +4253,10 @@modelspremium-featuresquery-processor+ settings+ taskutil+ util.configwarehouses.schemaenterprise/permissions.sandbox}:model-exports #{:model/ConnectionImpersonation}diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 8cc8395e20e..52a4f3b9606 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -19,6 +19,7 @@[metabase-enterprise.database-replication.init][metabase-enterprise.dependencies.init][metabase-enterprise.gsheets.init]+ [metabase-enterprise.impersonation.init][metabase-enterprise.mcp.init][metabase-enterprise.memoize-monitor.init][metabase-enterprise.metabot.init]diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/db.clj b/enterprise/backend/src/metabase_enterprise/impersonation/db.cljindex 4dbd0765ddb..e9c1311e5b8 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/db.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/db.clj@@ -3,7 +3,9 @@additional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."(:require[malli.util :as mut]+ [metabase-enterprise.impersonation.models.impersonation-query-cache :as impersonation-query-cache][metabase-enterprise.impersonation.schema :as impersonation.schema]+ [metabase.app-db.core :as app-db][metabase.lib.schema.id :as lib.schema.id][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -87,3 +89,35 @@current user, such as an internal or unauthenticated context)."[user-id :- [:maybe ::lib.schema.id/user]](t2/select-fn-set :group_id :model/PermissionsGroupMembership :user_id user-id))++(mu/defn query-cache :- [:maybe ::impersonation.schema/impersonation-query-cache]+ "The ImpersonationQueryCache with the lookup key of `input`, or nil."+ [{:keys [input_query input_engine input_allow_write input_admin input_version]}+ :- ::impersonation.schema/impersonation-query-cache.update]+ (t2/select-one :model/ImpersonationQueryCache+ :input_query_hash (impersonation-query-cache/query-hash input_query)+ :input_engine input_engine+ :input_allow_write input_allow_write+ :input_admin input_admin+ :input_version input_version))++(mu/defn update-or-insert-query-cache! :- ms/PositiveInt+ "Update the ImpersonationQueryCache with the lookup key of `row` to `row`, or insert `row` when there is none."+ [{:keys [input_query input_engine input_allow_write input_admin input_version] :as row}+ :- ::impersonation.schema/impersonation-query-cache.update]+ (app-db/update-or-insert! :model/ImpersonationQueryCache+ {:input_query_hash (impersonation-query-cache/query-hash input_query)+ :input_engine input_engine+ :input_allow_write input_allow_write+ :input_admin input_admin+ :input_version input_version}+ (constantly row)))++(mu/defn delete-outdated-query-caches! :- :int+ "Delete the ImpersonationQueryCaches cached by a Metabase version other than `version` or before `cutoff`,+ returning the number deleted."+ [version :- :string+ cutoff :- (ms/InstanceOfClass java.time.OffsetDateTime)]+ (t2/delete! :model/ImpersonationQueryCache {:where [:or+ [:not= :input_version version]+ [:< :created_at cutoff]]}))diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/init.clj b/enterprise/backend/src/metabase_enterprise/impersonation/init.cljnew file mode 100644index 00000000000..f22af04111f--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/impersonation/init.clj@@ -0,0 +1,4 @@+(ns metabase-enterprise.impersonation.init+ (:require+ [metabase-enterprise.impersonation.settings]+ [metabase-enterprise.impersonation.task.query-cache-cleanup]))diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.cljindex 90315e0191e..6959ab92930 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj@@ -1,17 +1,21 @@(ns metabase-enterprise.impersonation.middleware(:require[metabase-enterprise.impersonation.driver :as impersonation.driver]+ [metabase-enterprise.impersonation.query-cache :as query-cache][metabase.api.common :as api][metabase.driver :as driver]+ [metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema][metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.interface :as qp.i];; legacy usage -- don't do things like this going forward^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]- [metabase.util.i18n :refer [tru]]))+ [metabase.util.i18n :refer [tru]]+ [metabase.util.malli :as mu]))(defn- validates-native-queries?"Does `driver` supply its own `driver/validate-impersonated-query`?"@@ -28,14 +32,35 @@(throw (ex-info (tru "Connection impersonation does not support native queries on this database."){:type qp.error-type/invalid-query}))))+(mu/defn- validate-native-stage :- ::lib.schema/stage.native+ "Validate the native `stage` of `query` for `driver` on its own, through the impersonation query cache."+ [driver :- :keyword+ query :- ::lib.schema/query+ stage :- ::lib.schema/stage.native]+ (let [admin? api/*is-superuser?*]+ (assoc stage :native+ (query-cache/validated-query+ driver+ driver.settings/*impersonation-allow-write?*+ admin?+ (:native stage)+ #(-> (driver/validate-impersonated-query+ driver+ (cond-> (assoc query :stages [stage]) admin? (assoc :impersonation/admin? true)))+ :stages+ first+ :native)))))+(defn- validate-impersonated-query- "Enforce the single-statement rule on `query` for `driver`."+ "Enforce the single-statement rule on every native stage of `query` for `driver`, including stages nested in joins."[driver database query](if (driver.u/supports? driver :connection-impersonation database)(do (assert-native-validation-supported driver query)- (driver/validate-impersonated-query- driver- (cond-> query api/*is-superuser?* (assoc :impersonation/admin? true))))+ (lib/walk-stages+ query+ (fn [_query _path stage]+ (when (lib/native-stage? stage)+ (validate-native-stage driver query stage)))))query))(defenterprise apply-impersonationdiff --git a/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.clj b/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.cljnew file mode 100644index 00000000000..2350a16130d--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/impersonation/models/impersonation_query_cache.clj@@ -0,0 +1,35 @@+(ns metabase-enterprise.impersonation.models.impersonation-query-cache+ "Model definition for ImpersonationQueryCaches, the cached validation results of native query stages run on+ impersonated databases."+ (:require+ [buddy.core.codecs :as codecs]+ [buddy.core.hash :as buddy-hash]+ [metabase.models.interface :as mi]+ [metabase.util.malli :as mu]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/ImpersonationQueryCache [_model] :impersonation_query_cache)++(doto :model/ImpersonationQueryCache+ (derive :metabase/model)+ (derive :hook/created-at-timestamped?))++(t2/deftransforms :model/ImpersonationQueryCache+ {:input_engine mi/transform-keyword+ :result mi/transform-keyword})++(mu/defn query-hash :- :string+ "The SHA-256 hex of the native SQL `query`, which looks an ImpersonationQueryCache up."+ [query :- :string]+ (codecs/bytes->hex (buddy-hash/sha256 ^String query)))++(t2/define-before-insert :model/ImpersonationQueryCache+ [row]+ (assoc row :input_query_hash (query-hash (:input_query row))))++(t2/define-before-update :model/ImpersonationQueryCache+ [row]+ (let [changes (t2/changes row)]+ (cond-> row+ (contains? changes :input_query) (assoc :input_query_hash (query-hash (:input_query changes))))))diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.clj b/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.cljnew file mode 100644index 00000000000..31cccb97ded--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/impersonation/query_cache.clj@@ -0,0 +1,82 @@+(ns metabase-enterprise.impersonation.query-cache+ "Caches the result of validating each native stage of an impersonated query in the app DB, since validation parses+ the SQL and can take seconds."+ (:require+ [metabase-enterprise.impersonation.db :as impersonation.db]+ [metabase-enterprise.impersonation.schema :as impersonation.schema]+ [metabase.config.core :as config]+ [metabase.driver.sql :as driver.sql]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu])+ (:import+ (clojure.lang ExceptionInfo)))++(set! *warn-on-reflection* true)++(mu/defn- query-cache-input :- ::impersonation.schema/impersonation-query-cache.update+ "The lookup key of validating the native SQL `native` for `engine`."+ [engine :- :keyword+ allow-write? :- [:maybe :boolean]+ admin? :- [:maybe :boolean]+ native :- :string]+ {:input_version config/mb-version-string+ :input_engine engine+ :input_allow_write (boolean allow-write?)+ :input_admin (boolean admin?)+ :input_query native})++(mu/defn- cached-result :- [:maybe ::impersonation.schema/impersonation-query-cache]+ "The cached result for `input`, or nil when there is none or the cache cannot be read."+ [input :- ::impersonation.schema/impersonation-query-cache.update]+ (try+ (let [row (impersonation.db/query-cache input)]+ (when (= (:input_query input) (:input_query row))+ row))+ (catch Throwable e+ (log/errorf "Error reading the impersonation query cache: %s" (ex-message e))+ nil)))++(mu/defn- save-result! :- :nil+ "Cache `result` for `input`, replacing a row with the same lookup key, or log why the cache cannot be written."+ [input :- ::impersonation.schema/impersonation-query-cache.update+ result :- ::impersonation.schema/impersonation-query-cache.update]+ (try+ (impersonation.db/update-or-insert-query-cache! (merge input result))+ (catch Throwable e+ (log/errorf "Error writing the impersonation query cache: %s" (ex-message e))))+ nil)++(mu/defn- replay-result :- :string+ "The validated SQL of the cached `result` for the native SQL `native`, or the refusal it records thrown."+ [{:keys [result result_query result_reason]} :- ::impersonation.schema/impersonation-query-cache+ native :- :string+ allow-write? :- [:maybe :boolean]]+ (if (= result :valid)+ result_query+ (throw (driver.sql/impersonated-query-refusal result native allow-write? result_reason))))++(mu/defn- validate-and-save! :- :string+ "Call `validate` and cache what it returns, or the refusal it throws when that refusal records its result."+ [input :- ::impersonation.schema/impersonation-query-cache.update+ validate :- fn?]+ (try+ (let [validated (validate)]+ (save-result! input {:result :valid, :result_query validated})+ validated)+ (catch ExceptionInfo e+ (when-let [result (#{:unparseable :unvalidatable :invalid} (:impersonation/result (ex-data e)))]+ (save-result! input {:result result, :result_reason (:reason (ex-data e))}))+ (throw e))))++(mu/defn validated-query :- :string+ "The validated SQL to run in place of the native stage SQL `native` on `engine`, from the cache or else by calling+ `validate`, which returns that SQL or throws a refusal."+ [engine :- :keyword+ allow-write? :- [:maybe :boolean]+ admin? :- [:maybe :boolean]+ native :- :string+ validate :- fn?]+ (let [input (query-cache-input engine allow-write? admin? native)]+ (if-let [cached (cached-result input)]+ (replay-result cached native allow-write?)+ (validate-and-save! input validate))))diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj b/enterprise/backend/src/metabase_enterprise/impersonation/schema.cljindex 6c2f8203855..d06ee8d4605 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/schema.clj@@ -18,3 +18,28 @@[:db_id {:optional true} [:maybe ::lib.schema.id/database]][:group_id {:optional true} [:maybe ms/PositiveInt]][:attribute {:optional true} [:maybe :string]]])++(mr/def ::impersonation-query-cache.result+ "The outcome of validating an impersonated native query, as an ImpersonationQueryCache records it."+ [:enum :valid :unparseable :unvalidatable :invalid])++(mr/def ::impersonation-query-cache+ "An ImpersonationQueryCache as selected from the app DB: every column of `:impersonation_query_cache`."+ [:merge+ ::impersonation-query-cache.update+ [:map {:closed true}+ [:id ms/PositiveInt]+ [:input_query_hash :string]]])++(mr/def ::impersonation-query-cache.update+ "What an update (or insert) of an ImpersonationQueryCache accepts: every column of `:impersonation_query_cache` except `id`, all optional."+ [:map {:closed true}+ [:input_version {:optional true} :string]+ [:input_engine {:optional true} :keyword]+ [:input_allow_write {:optional true} :boolean]+ [:input_admin {:optional true} :boolean]+ [:input_query {:optional true} :string]+ [:result {:optional true} ::impersonation-query-cache.result]+ [:result_query {:optional true} [:maybe :string]]+ [:result_reason {:optional true} [:maybe :string]]+ [:created_at {:optional true} (ms/InstanceOfClass java.time.temporal.Temporal)]])diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/settings.clj b/enterprise/backend/src/metabase_enterprise/impersonation/settings.cljnew file mode 100644index 00000000000..672a1df97b4--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/impersonation/settings.clj@@ -0,0 +1,13 @@+(ns metabase-enterprise.impersonation.settings+ (:require+ [metabase.settings.core :refer [defsetting]]+ [metabase.util.i18n :refer [deferred-tru]]))++(defsetting impersonation-query-cache-retention-days+ (deferred-tru "Cached validation results of impersonated native queries are deleted once they are this many days old.")+ :encryption :no+ :visibility :internal+ :default 30+ :type :positive-integer+ :export? false+ :doc false)diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.clj b/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.cljnew file mode 100644index 00000000000..b69759ba879--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/impersonation/task/query_cache_cleanup.clj@@ -0,0 +1,45 @@+(ns metabase-enterprise.impersonation.task.query-cache-cleanup+ "Scheduled task that deletes impersonation query cache rows cached by another Metabase version or more than+ [[impersonation.settings/impersonation-query-cache-retention-days]] ago."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase-enterprise.impersonation.db :as impersonation.db]+ [metabase-enterprise.impersonation.settings :as impersonation.settings]+ [metabase.config.core :as config]+ [metabase.task.core :as task]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++(def ^:private cleanup-job-key (jobs/key "metabase.task.impersonation.query-cache-cleanup.job"))+(def ^:private cleanup-trigger-key (triggers/key "metabase.task.impersonation.query-cache-cleanup.trigger"))++(mu/defn- cleanup-query-cache! :- :nil+ "Delete the outdated impersonation query cache rows."+ []+ (let [cutoff (t/minus (t/offset-date-time) (t/days (impersonation.settings/impersonation-query-cache-retention-days)))+ deleted (impersonation.db/delete-outdated-query-caches! config/mb-version-string cutoff)]+ (log/infof "Impersonation query cache cleanup complete. Deleted %d row(s)." deleted)))++(task/defjob ImpersonationQueryCacheCleanup+ "Delete outdated impersonation query cache rows"+ {:saved-class "metabase_enterprise.impersonation.task.query_cache_cleanup.ImpersonationQueryCacheCleanup"+ :concurrent? false}+ [_ctx]+ (cleanup-query-cache!))++(defmethod task/init! ::ImpersonationQueryCacheCleanup+ [_]+ (let [job (jobs/build+ (jobs/of-type ImpersonationQueryCacheCleanup)+ (jobs/with-identity cleanup-job-key))+ trigger (triggers/build+ (triggers/with-identity cleanup-trigger-key)+ (triggers/start-now)+ (triggers/with-schedule+ (cron/cron-schedule "37 41 23 * * ?")))]+ (task/schedule-task! job trigger)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex dea7f19b834..d008da9317b 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -111,6 +111,7 @@"ExplorationThread""ExplorationThreadTimeline""ImplicitAction"+ "ImpersonationQueryCache""LoginHistory""McpFeedback""McpGroupPermission"diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.cljindex de84b678d69..9bdce39d5ff 100644--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj@@ -7,6 +7,7 @@[clojure.test :refer :all][metabase-enterprise.impersonation.driver :as impersonation.driver][metabase-enterprise.impersonation.middleware :as impersonation.middleware]+ [metabase-enterprise.impersonation.query-cache :as query-cache][metabase-enterprise.impersonation.util-test :as impersonation.util-test][metabase-enterprise.test :as met][metabase.actions.execution :as actions.execution]@@ -36,6 +37,7 @@[toucan2.core :as t2][toucan2.tools.with-temp :as t2.with-temp])(:import+ (clojure.lang ExceptionInfo)(com.mchange.v2.c3p0 PooledDataSource)(java.sql Connection)(java.util.concurrent CountDownLatch)))@@ -1181,6 +1183,32 @@"SELECT set_config('role', 'none', false); DROP TABLE table""DO $$ BEGIN EXECUTE 'SET ROLE NONE; DROP TABLE table'; END $$;"))))))))))+(deftest cached-refusal-fails-query-like-uncached-refusal-test+ (mt/test-drivers (mt/normal-drivers-with-feature :connection-impersonation)+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (impersonation.util-test/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]+ :attributes {"impersonation_attr" "impersonation_role"}}+ (let [run-query (fn [sql]+ (try+ (qp/process-query (lib/native-query (mt/metadata-provider) sql))+ (catch ExceptionInfo e+ {:message (ex-message e)+ :refusal-message (ex-message (ex-cause e))+ :refusal-data (ex-data (ex-cause e))})))]+ (doseq [[result sql] [[:unparseable "SELECT ("]+ [:invalid "SELECT 1; SELECT 2"]+ [:unvalidatable (str "SELECT 1 WHERE 1 IN (" (str/join ", " (range 3000)) ")")]]]+ (testing result+ (let [sql (str "/* " (mt/random-name) " */ " sql)+ uncached (mt/with-dynamic-fn-redefs [query-cache/validated-query (fn [_engine _allow-write? _admin? _native validate]+ (validate))]+ (run-query sql))]+ (is (= result (get-in uncached [:refusal-data :impersonation/result])))+ (is (= uncached (run-query sql)))+ (is (t2/exists? :model/ImpersonationQueryCache :input_query sql))+ (is (= uncached (run-query sql))))))))))))+(deftest reject-non-select-statements-for-admin-on-impersonated-db-test(testing "Admin users are also validated on impersonated databases"(mt/test-drivers (mt/normal-drivers-with-feature :connection-impersonation)diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.cljnew file mode 100644index 00000000000..93b0724732c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/impersonation/query_cache_test.clj@@ -0,0 +1,106 @@+(ns metabase-enterprise.impersonation.query-cache-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.impersonation.models.impersonation-query-cache :as impersonation-query-cache]+ [metabase-enterprise.impersonation.query-cache :as query-cache]+ [metabase-enterprise.impersonation.task.query-cache-cleanup :as query-cache-cleanup]+ [metabase.config.core :as config]+ [metabase.driver.sql :as driver.sql]+ [metabase.test :as mt]+ [toucan2.core :as t2])+ (:import+ (clojure.lang ExceptionInfo)))++(deftest valid-result-is-cached-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)+ calls (atom 0)+ validate (fn [] (swap! calls inc) "SELECT 1")]+ (is (= "SELECT 1" (query-cache/validated-query :postgres false false native validate)))+ (is (= "SELECT 1" (query-cache/validated-query :postgres false false native validate)))+ (is (= 1 @calls)))))++(deftest refusal-with-result-is-cached-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (doseq [[result allow-write? reason message] [[:unparseable false nil #"Unable to parse native query"]+ [:unvalidatable false "large-literal-list" #"Unable to validate this native query"]+ [:invalid false nil #"Must be a single select statement"]+ [:invalid true nil #"Must be a single insert, update, or delete statement"]]]+ (testing (str result " allow-write? " allow-write?)+ (let [native (mt/random-name)+ refusal (driver.sql/impersonated-query-refusal result native allow-write? reason)]+ (is (thrown-with-msg? ExceptionInfo message+ (query-cache/validated-query :postgres allow-write? false native #(throw refusal))))+ (is (thrown-with-msg? ExceptionInfo message+ (query-cache/validated-query :postgres allow-write? false native (constantly "SELECT 1"))))+ (is (= (ex-data refusal)+ (try+ (query-cache/validated-query :postgres allow-write? false native (constantly "SELECT 1"))+ (catch ExceptionInfo e+ (ex-data e))))))))))++(deftest refusal-without-result-is-not-cached-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)+ calls (atom 0)+ validate (fn [] (swap! calls inc) (throw (ex-info "Boom" {})))]+ (is (thrown-with-msg? ExceptionInfo #"Boom" (query-cache/validated-query :postgres false false native validate)))+ (is (thrown-with-msg? ExceptionInfo #"Boom" (query-cache/validated-query :postgres false false native validate)))+ (is (= 2 @calls)))))++(deftest every-input-is-part-of-key-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)]+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))+ (is (= "SELECT 2" (query-cache/validated-query :mysql false false native (constantly "SELECT 2"))))+ (is (= "SELECT 2" (query-cache/validated-query :postgres true false native (constantly "SELECT 2"))))+ (is (= "SELECT 2" (query-cache/validated-query :postgres false true native (constantly "SELECT 2"))))+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false (str native " ") (constantly "SELECT 2")))))))++(deftest other-version-is-not-read-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)]+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))+ (t2/update! :model/ImpersonationQueryCache :input_query native {:input_version "v0.0.0 (other)"})+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false native (constantly "SELECT 2")))))))++(deftest hash-collision-is-replaced-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)]+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))+ (t2/update! :impersonation_query_cache :input_query native {:input_query "SELECT 'colliding'"})+ (testing "row whose input_query differs is not used, and replacing it keeps surrounding transaction usable"+ (t2/with-transaction [_conn]+ (is (= "SELECT 2" (query-cache/validated-query :postgres false false native (constantly "SELECT 2"))))+ (is (= "SELECT 2" (t2/select-one-fn :result_query :model/ImpersonationQueryCache :input_query native))))))))++(deftest hash-follows-input-query-test+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [native (mt/random-name)+ changed (mt/random-name)]+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1"))+ (is (= (impersonation-query-cache/query-hash native)+ (t2/select-one-fn :input_query_hash :model/ImpersonationQueryCache :input_query native)))+ (t2/update! :model/ImpersonationQueryCache :input_query native {:input_query changed})+ (is (= (impersonation-query-cache/query-hash changed)+ (t2/select-one-fn :input_query_hash :model/ImpersonationQueryCache :input_query changed))))))++(deftest cleanup-deletes-outdated-rows-test+ (mt/with-temporary-setting-values [impersonation-query-cache-retention-days 7]+ (mt/with-model-cleanup [:model/ImpersonationQueryCache]+ (let [current (mt/random-name)+ recent (mt/random-name)+ old (mt/random-name)+ outdated (mt/random-name)]+ (doseq [native [current recent old outdated]]+ (query-cache/validated-query :postgres false false native (constantly "SELECT 1")))+ (t2/update! :model/ImpersonationQueryCache :input_query recent {:created_at (t/minus (t/offset-date-time) (t/days 6))})+ (t2/update! :model/ImpersonationQueryCache :input_query old {:created_at (t/minus (t/offset-date-time) (t/days 8))})+ (t2/update! :model/ImpersonationQueryCache :input_query outdated {:input_version "v0.0.0 (other)"})+ (#'query-cache-cleanup/cleanup-query-cache!)+ (is (= #{current recent}+ (t2/select-fn-set :input_query :model/ImpersonationQueryCache+ :input_query [:in [current recent old outdated]])))+ (is (= config/mb-version-string+ (t2/select-one-fn :input_version :model/ImpersonationQueryCache :input_query current)))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 0d35cd6f241..cbb870e2e70 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -156,6 +156,7 @@:model/ViewLog:model/Sandbox:model/ConnectionImpersonation+ :model/ImpersonationQueryCache:model/SecurityAdvisory:model/CloudMigration:model/Commentdiff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yamlindex 0d35b65d4e3..31946f301a4 100644--- a/resources/migrations/058_update_migrations.yaml+++ b/resources/migrations/058_update_migrations.yaml@@ -1692,6 +1692,94 @@ databaseChangeLog:tableName: core_sessioncolumnName: saml_name_id_format+ - changeSet:+ id: v58.2026-10-29T00:00:00+ author: ranquild+ comment: Add impersonation_query_cache+ changes:+ - createTable:+ tableName: impersonation_query_cache+ remarks: Cached validation results of native query stages run on impersonated databases+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ primaryKeyName: pk_impersonation_query_cache+ - column:+ name: input_version+ type: varchar(255)+ remarks: Metabase version that validated the query; rows from other versions are never read+ constraints:+ nullable: false+ - column:+ name: input_engine+ type: varchar(254)+ remarks: Driver that validated the query+ constraints:+ nullable: false+ - column:+ name: input_allow_write+ type: ${boolean.type}+ remarks: Whether the query was validated as a write statement rather than a select+ constraints:+ nullable: false+ - column:+ name: input_admin+ type: ${boolean.type}+ remarks: Whether the query was validated for an admin, who is allowed more than other users+ constraints:+ nullable: false+ - column:+ name: input_query_hash+ type: varchar(64)+ remarks: SHA-256 hex of input_query, used to look the row up+ constraints:+ nullable: false+ - column:+ name: input_query+ type: ${text.type}+ remarks: Native SQL of the stage after parameter substitution+ constraints:+ nullable: false+ - column:+ name: result+ type: varchar(32)+ remarks: 'Validation outcome: valid, unparseable, unvalidatable or invalid'+ constraints:+ nullable: false+ - column:+ name: result_query+ type: ${text.type}+ remarks: Validated SQL to run in place of input_query, set only when result is valid+ constraints:+ nullable: true+ - column:+ name: result_reason+ type: varchar(64)+ remarks: Why the validator could not check the query, when it says+ constraints:+ nullable: true+ - column:+ name: created_at+ type: ${timestamp_type}+ remarks: Timestamp when the result was cached+ constraints:+ nullable: false++ - changeSet:+ id: v58.2026-10-29T00:00:01+ author: ranquild+ comment: Add the lookup key of impersonation_query_cache+ changes:+ - addUniqueConstraint:+ tableName: impersonation_query_cache+ columnNames: input_query_hash, input_engine, input_allow_write, input_admin, input_version+ constraintName: uq_impersonation_query_cache_input+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex a8e637741fb..f42c7c7c2af 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -337,10 +337,28 @@native-query :- :metabase.lib.schema/native-only-query](sql-tools/validate-query driver native-query))+(mu/defn impersonated-query-refusal :- [:fn #(instance? clojure.lang.ExceptionInfo %)]+ "The [[metabase.query-processor.error-type/invalid-query]] `ex-info` refusing the impersonated native `sql`+ with `result`: `:unparseable`, `:unvalidatable` (with the validator's `reason`) or `:invalid`. `write?` says+ whether the query had to be a write statement rather than a select."+ [result :- [:enum :unparseable :unvalidatable :invalid]+ sql :- :string+ write? :- [:maybe :boolean]+ reason :- [:maybe :string]]+ (ex-info (case result+ :unparseable (tru "Unable to parse native query. There might be something wrong with your query.")+ :unvalidatable (tru "Unable to validate this native query on an impersonated database. Reduce the size of any large VALUES, IN, or ARRAY lists.")+ :invalid (if write?+ (tru "Invalid impersonated native query. Must be a single insert, update, or delete statement.")+ (tru "Invalid impersonated native query. Must be a single select statement.")))+ (cond-> {:type qp.error-type/invalid-query+ :impersonation/result result+ :sql sql}+ reason (assoc :reason reason))))+(defn- validate-impersonated-native-stage"Validate one native `stage` for `driver`, returning it with `:native` replaced by the reconstructed,- validated SQL — or throwing a [[metabase.query-processor.error-type/invalid-query]] `ex-info` when it must- be refused.+ validated SQL — or throwing an [[impersonated-query-refusal]] when it must be refused.`admin?` users run as the full connection user and are never actually impersonated."[driver stage admin?]@@ -352,9 +370,7 @@error(do(log/warnf "Failed to parse native query: %s" error)- (throw (ex-info (tru "Unable to parse native query. There might be something wrong with your query.")- {:type qp.error-type/invalid-query- :sql (:native stage)})))+ (throw (impersonated-query-refusal :unparseable (:native stage) write? nil)));; The SQL was not checked at all, as opposed to checked and found to be two statements. Admins run as;; the full connection user and are never actually impersonated, so validating their native SQL@@ -365,10 +381,7 @@stage(do(log/warnf "Refusing impersonated native query on %s: cannot validate (%s)" driver reason)- (throw (ex-info (tru "Unable to validate this native query on an impersonated database. Reduce the size of any large VALUES, IN, or ARRAY lists.")- {:type qp.error-type/invalid-query- :reason reason- :sql (:native stage)}))))+ (throw (impersonated-query-refusal :unvalidatable (:native stage) write? reason))));; Admins may also run a single statement that is not of the required type (e.g. `SHOW TIMEZONE`),;; but never more than one statement.@@ -376,11 +389,7 @@(assoc stage :native sql):else- (throw (ex-info (if write?- (tru "Invalid impersonated native query. Must be a single insert, update, or delete statement.")- (tru "Invalid impersonated native query. Must be a single select statement."))- {:type qp.error-type/invalid-query- :sql (:native stage)})))))+ (throw (impersonated-query-refusal :invalid (:native stage) write? nil)))))(defn validate-impersonated-query*"Validates a native query by parsing it and ensuring that it is a single statement.diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 78000580a6f..61d70329a39 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -69,6 +69,7 @@:model/Glossary metabase.glossary.models.glossary:model/Sandbox metabase-enterprise.sandbox.models.sandbox:model/ImplicitAction metabase.actions.models+ :model/ImpersonationQueryCache metabase-enterprise.impersonation.models.impersonation-query-cache:model/LoginHistory metabase.login-history.models.login-history:model/McpFeedback metabase.mcp.models.mcp-feedback:model/McpGroupPermission metabase.mcp.models.mcp-group-permissiondiff --git a/test/metabase/app_db/quartz_test.clj b/test/metabase/app_db/quartz_test.cljindex 17a89cdb748..f19288bb293 100644--- a/test/metabase/app_db/quartz_test.clj+++ b/test/metabase/app_db/quartz_test.clj@@ -178,6 +178,7 @@"metabase_enterprise.dependencies.task.entity_check.DependencyEntityCheck""metabase_enterprise.dependencies.task.result_metadata.UpdateResultMetadata""metabase_enterprise.entity_retrieval.task.sync.OsiAiContextSync"+ "metabase_enterprise.impersonation.task.query_cache_cleanup.ImpersonationQueryCacheCleanup""metabase_enterprise.mcp.task.mcp_usage_trimmer.McpUsageTrimmer""metabase_enterprise.metabot.task.ai_usage_trimmer.AiUsageTrimmer""metabase_enterprise.remote_sync.task.import.AutoImport"diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex efe7ece5f8a..e790203758e 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -153,6 +153,7 @@:model/DependencyStatus:model/EmbeddingSdkReact18Host:model/ExplorationQueryResult+ :model/ImpersonationQueryCache:model/McpQueryHandle:model/McpSessionLog:model/McpToolCallLog