The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-3192] audit every action execution in query_execution (#82257)
YB9e27498bbf68
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f6152d616e3..178341a7dd9 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -106,6 +106,7 @@model-persistencemodelsparameters+ permissionspublic-sharingqueriesquery-processor@@ -121,6 +122,7 @@:model/DashboardCard:model/Database:model/Field+ :model/QueryExecution:model/Table:model/User}}diff --git a/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj b/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.cljindex 8681564ad93..64ff8773935 100644--- a/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj+++ b/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj@@ -1,7 +1,12 @@(ns metabase-enterprise.action-v2.actions-test(:require[clojure.test :refer :all]- [metabase-enterprise.action-v2.actions :as action-v2.actions]))+ [metabase-enterprise.action-v2.actions :as action-v2.actions]+ [metabase-enterprise.action-v2.test-util :as action-v2.tu]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]))++(use-fixtures :once (fixtures/initialize :test-users-personal-collections))(def unsupported-dbs-msg #'action-v2.actions/unsupported-dbs-msg)@@ -10,3 +15,31 @@(is (= "Data editing isn't supported on one of the target databases." (unsupported-dbs-msg [{:id 1} {:id 2}] [{:id 1}])))(is (= "Data editing isn't supported on the target databases." (unsupported-dbs-msg [{:id 1} {:id 2}] [{:id 1} {:id 2}])))(is (= "Data editing isn't supported on some of the target databases." (unsupported-dbs-msg [{:id 1} {:id 2} {:id 3}] [{:id 1} {:id 2}]))))++(deftest data-grid-audit-trail-test+ (testing "a data-grid write and its undo each record exactly one QueryExecution row"+ (mt/with-empty-h2-app-db!+ (mt/with-premium-features #{:table-data-editing}+ (action-v2.tu/with-test-tables! [table-id [{:id [:int]+ :name [:text]}+ {:primary-key [:id]}]]+ (let [user-id (mt/user->id :crowberto)+ since (mt/latest-query-execution-id)]+ (action-v2.tu/create-rows! table-id user-id 200 [{:id 1 :name "Snorkmaiden"}])+ (let [rows (mt/action-executions since)]+ (is (= 1 (count rows)))+ (testing "a data-grid action has no Action row to point at"+ (is (=? {:context :action-execute+ :native false+ :action_id nil+ :result_rows 1+ :error nil+ :database_id (mt/id)}+ (first rows)))))+ (testing "undo runs its nested actions under one row"+ (let [since (mt/latest-query-execution-id)]+ (mt/user-http-request user-id :post "/ee/action-v2/execute-bulk"+ {:action :data-editing/undo+ :scope {:table-id table-id}+ :inputs [{}]})+ (is (= 1 (count (mt/action-executions since))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.cljindex 7e5fbcd23e7..54cfdc4fa54 100644--- a/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj+++ b/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj@@ -6,6 +6,8 @@(:require[clojure.test :refer :all][metabase-enterprise.impersonation.util-test :as impersonation.util-test]+ [metabase.actions.execution :as actions.execution]+ [metabase.actions.models :as action][metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.pipeline :as qp.pipeline]@@ -54,3 +56,48 @@"QueryExecution row for a FAILED impersonated query should still record is_impersonated=true")(is (some? (:error (latest-query-execution)))"Sanity check: the QueryExecution row should have an error message")))))))++(deftest action-row-records-impersonation-test+ (testing "a native action run under an impersonation policy records is_impersonated, on success and on failure"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{ok-action-id :action-id} {:type :query}+ {bad-action-id :action-id} {:type :query+ :parameters []+ :dataset_query (mt/native-query+ {:query "UPDATE categories SET name = 1/0 WHERE id = 1"})}]+ (impersonation.util-test/with-impersonations!+ {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]+ :attributes {"impersonation_attr" "impersonation_role"}}+ (testing "success"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! (action/select-action :id ok-action-id) {"id" 1 "name" "Bird"})+ (is (=? {:is_impersonated true, :error nil}+ (first (mt/action-executions since))))))+ (testing "failure"+ (let [since (mt/latest-query-execution-id)]+ (is (thrown? clojure.lang.ExceptionInfo+ (actions.execution/execute-action! (action/select-action :id bad-action-id) {})))+ (is (=? {:is_impersonated true, :error some?}+ (first (mt/action-executions since))))))))))))++(deftest implicit-action-row-records-impersonation-test+ (testing "an implicit action records is_impersonated from the user's policy on the database, as reads do"+ (mt/with-premium-features #{:advanced-permissions}+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :implicit :kind "row/update"}]+ (impersonation.util-test/with-impersonations!+ {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]+ :attributes {"impersonation_attr" "impersonation_role"}}+ ;; H2 can't switch roles, so a true here comes from the policy, as it does for reads on H2+ (testing "impersonated user"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})+ (is (=? {:is_impersonated true, :native false, :error nil}+ (first (mt/action-executions since))))))+ (testing "admins are never impersonated"+ (mt/with-test-user :crowberto+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})+ (is (=? {:is_impersonated false, :error nil}+ (first (mt/action-executions since)))))))))))))diff --git a/src/metabase/actions/actions.clj b/src/metabase/actions/actions.cljindex e1515b34f88..49910307588 100644--- a/src/metabase/actions/actions.clj+++ b/src/metabase/actions/actions.clj@@ -3,6 +3,7 @@(:require[malli.error :as me][metabase.actions.args :as actions.args]+ [metabase.actions.audit :as actions.audit][metabase.actions.db :as actions.db][metabase.actions.events :as actions.events][metabase.actions.hierarchy :as actions.hierarchy]@@ -259,7 +260,10 @@[actionscopearg-map-or-maps- & {:keys [policy existing-context user-id]}]+ ;; `action-id`, `dashboard-id` and `audit-context` are attribution for the audit row; the scope maps are closed+ ;; schemas, so they ride along as kwargs instead.+ & {:keys [policy existing-context user-id action-id dashboard-id]+ audit-context :context}](when (and existing-context user-id)(assert (= user-id (:user-id existing-context)) "Existing context has a consistent user-id"))(log/with-context {:action action}@@ -308,21 +312,37 @@(check-data-editing-enabled-for-database! db))))(log/with-context {:db-id (:id db)}(binding [*misc-value-cache* (atom {:databases (zipmap (map :id dbs) dbs)})]- (check-permissions policy arg-maps)- (let [result (let [context (-> existing-context- ;; TODO fix tons of tests which execute without user scope- (u/assoc-default :user-id (identity #_api/check-500- (or user-id api/*current-user-id*)))- (u/assoc-default :scope scope))]- (if-not driver- (perform-action-internal! action-kw context arg-maps)- (driver/with-driver driver- (let [context (assoc context- ;; Legacy drivers dispatch on this, for now.- ;; TODO As far as I'm aware we only have :sql-jdbc defined actions, so can stop dispatching- ;; on this and just fail if the dynamically determined driver is incompatible.- :driver driver)]- (perform-action-internal! action-kw context arg-maps)))))]+ (let [context (-> existing-context+ ;; TODO fix tons of tests which execute without user scope+ (u/assoc-default :user-id (identity #_api/check-500+ (or user-id api/*current-user-id*)))+ (u/assoc-default :scope scope))+ ;; the permission check is inside the audited span so that a denial is traced the same way a+ ;; driver error is+ result (actions.audit/with-audited-execution+ {:action action-kw+ :action-id action-id+ :dashboard-id dashboard-id+ :database-id (:id db)+ :user-id (:user-id context)+ :context (or audit-context :action-execute)+ :native? false+ :template {:type :internal+ :action (u/qualified-name action-kw)+ :database (:id db)+ :scope (actions.scope/normalize-scope scope)}+ :inputs arg-maps}+ (fn [{:keys [outputs]}] {:result_rows (count outputs)})+ (check-permissions policy arg-maps)+ (if-not driver+ (perform-action-internal! action-kw context arg-maps)+ (driver/with-driver driver+ (let [context (assoc context+ ;; Legacy drivers dispatch on this, for now.+ ;; TODO As far as I'm aware we only have :sql-jdbc defined actions, so can stop dispatching+ ;; on this and just fail if the dynamically determined driver is incompatible.+ :driver driver)]+ (perform-action-internal! action-kw context arg-maps)))))]{:effects (:effects (:context result)):outputs (:outputs result)}))))))diff --git a/src/metabase/actions/audit.clj b/src/metabase/actions/audit.cljnew file mode 100644index 00000000000..da0eed0b604--- /dev/null+++ b/src/metabase/actions/audit.clj@@ -0,0 +1,113 @@+(ns metabase.actions.audit+ "Audit trail for action executions: one QueryExecution row per invocation, written synchronously on the request+ thread. A failed audit write is logged and never fails the action, so a destructive write goes untraced only when+ the app DB itself is down.++ The row's `context` (`:action-execute`, or `:public-action-execute` from the public endpoints) separates these+ write rows from the reads in `v_query_log`; `v_action_log` selects on it. Nested actions are not wrapped, so an+ invocation is one row."+ (:require+ [java-time.api :as t]+ [metabase.actions.db :as actions.db]+ [metabase.analytics.settings :as analytics.settings]+ [metabase.api.common :as api]+ [metabase.lib-be.core :as lib-be]+ [metabase.permissions.core :as perms]+ [metabase.queries.models.query :as query]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++(def ^:private Base+ "What a recording site knows about an invocation before it runs."+ [:map {:closed true}+ ;; e.g. :model.row/update, :query/execute+ [:action :keyword]+ [:action-id [:maybe pos-int?]]+ [:dashboard-id [:maybe pos-int?]]+ [:database-id [:maybe pos-int?]]+ [:user-id [:maybe pos-int?]]+ [:context [:enum :action-execute :public-action-execute]]+ [:native? :boolean]+ ;; hashed, and stored in `query` -- the SQL template or an action descriptor, never the input values+ [:template :map]+ ;; the input values that were actually supplied: PII-gated into `parameters`+ [:inputs [:sequential :any]]])++(defn- impersonated?+ "Whether an enforced connection-impersonation policy applies to the current user on `database-id`: the question the+ QP's impersonation preprocessing asks, so these rows agree with read rows. A lookup error (e.g. conflicting sandbox+ and impersonation policies) is logged and reads as false, so auditing never fails the action."+ [database-id]+ (boolean+ ;; public forms run with no user, and the OSS impl throws without one+ (when (and api/*current-user-id* database-id)+ (try+ (perms/impersonation-enforced-for-db? database-id)+ (catch Throwable e+ (log/warnf e "Could not resolve connection impersonation for database %d" database-id)+ false)))))++;; Mirrors `query-execution-info` in [[metabase.query-processor.middleware.process-userland-query]]; keep the two row+;; shapes in step when a column is added to `query_execution`.+(mu/defn- execution-row+ [{:keys [action-id dashboard-id database-id user-id context native? template inputs]} :- Base]+ {:action_id action-id+ :dashboard_id dashboard-id+ :database_id database-id+ :executor_id user-id+ :context context+ :native native?+ :hash (lib-be/query-hash template)+ ;; not a column: `save-queries-and-update-average-execution-times!` stores it in `query`, and it is dropped+ ;; before the insert, as the userland read path does+ :json_query template+ :parameterized (boolean (seq inputs))+ :parameters (when (and (seq inputs) (analytics.settings/analytics-pii-retention-enabled))+ (json/encode inputs))+ :tenant_id (:tenant_id @api/*current-user*)+ :started_at (t/zoned-date-time)+ ;; an action row can never be a cache hit; setting it keeps the EE cache-rerun join from ever matching+ :cache_hit false+ ;; native rows take both from the writeback QP's snapshot; other rows skip the QP, so ask the policy directly+ :is_impersonated (if native? false (impersonated? database-id))+ :is_db_routed false+ :result_rows 0+ :running_time 0})++(defn do-with-audited-execution+ "Run `thunk` and record one QueryExecution row for it, whether it succeeded or failed. `base` is a [[Base]] map;+ `result->row` turns the thunk's return value (nil when it threw) into extra columns, typically `:result_rows`.++ Recording errors are logged, never thrown: the warehouse transaction has already committed or rolled back, and a+ 500 after a destructive write invites a retry of it. The thunk's own exception is rethrown unchanged."+ [base result->row thunk]+ (let [start-ms (System/currentTimeMillis)+ row (execution-row base)+ record! (fn [extra-fn]+ (try+ (let [row (-> row+ (merge (extra-fn))+ (assoc :running_time (u/since-ms-wall-clock start-ms)))]+ (query/save-queries-and-update-average-execution-times!+ [{:query (:json_query row)+ :query-hash (:hash row)+ :execution-time-ms (:running_time row)}])+ (actions.db/insert-query-execution! (dissoc row :json_query)))+ (catch Throwable e+ (log/errorf e "Failed to record action execution for %s" (:action base)))))]+ (try+ (u/prog1 (thunk)+ (record! #(result->row <>)))+ (catch Throwable e+ (record! #(assoc (result->row nil) :error (or (some-> e ex-cause ex-message) (ex-message e))))+ (throw e)))))++(defmacro with-audited-execution+ "Macro form of [[do-with-audited-execution]]."+ {:style/indent 2}+ [base result->row & body]+ `(do-with-audited-execution ~base ~result->row (fn [] ~@body)))diff --git a/src/metabase/actions/db.clj b/src/metabase/actions/db.cljindex 7bb2fc58591..440645bb0c7 100644--- a/src/metabase/actions/db.clj+++ b/src/metabase/actions/db.clj@@ -4,6 +4,7 @@(:require[metabase.actions.schema :as actions.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.queries.schema :as queries.schema][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -83,6 +84,11 @@[action-id :- ::lib.schema.id/action](t2/delete! :model/DashboardCard :action_id action-id))+(mu/defn insert-query-execution!+ "Insert the QueryExecution `row` and return its id."+ [row :- ::queries.schema/query-execution.update]+ (t2/insert-returning-pk! :model/QueryExecution row))+(mu/defn insert-action!"Insert the Action `row` and return the inserted instance."[row :- ::actions.schema/action.for-insert]diff --git a/src/metabase/actions/execution.clj b/src/metabase/actions/execution.cljindex 0239ed92964..b27fc29f63e 100644--- a/src/metabase/actions/execution.clj+++ b/src/metabase/actions/execution.clj@@ -4,6 +4,7 @@[medley.core :as m][metabase.actions.actions :as actions][metabase.actions.args :as actions.args]+ [metabase.actions.audit :as actions.audit][metabase.actions.db :as actions.db][metabase.actions.http-action :as http-action][metabase.actions.models :as action]@@ -31,23 +32,40 @@"Execute a `QueryAction` with parameters as passed in from anendpoint of shape `{<parameter-id> <value>}`.- `action` should already be hydrated with its `:card`."+ `action` should already be hydrated with its `:card`. `opts` carries the audit attribution from the endpoint."[{query :dataset_query, model-id :model_id, :as action} :- [:map[:model_id ::lib.schema.id/card][:dataset_query ::lib.schema/native-only-query]]- request-parameters]+ request-parameters+ opts](log/tracef "Executing action for model %d" model-id)(driver.conn/with-write-connection(try- (let [parameters (for [parameter (:parameters action)]- ;; the query gets the parameter values, not the frontend's widget settings- (-> parameter- (select-keys [:id :type :target :slug :name :default :required :options])- (assoc :value (get request-parameters (:id parameter)))))- query (-> query- (assoc :parameters parameters))]+ (let [parameters (for [parameter (:parameters action)]+ ;; the query gets the parameter values, not the frontend's widget settings+ (-> parameter+ (select-keys [:id :type :target :slug :name :default :required :options])+ (assoc :value (get request-parameters (:id parameter)))))+ substituted-query (-> query+ (assoc :parameters parameters))+ ;; the routed destination database and the impersonation flag are only knowable from inside the+ ;; writeback QP's middleware stack+ execution-context (volatile! nil)](binding [qp.perms/*card-id* model-id]- (qp/execute-write-query! query)))+ (actions.audit/with-audited-execution+ {:action :query/execute+ :action-id (:id action)+ :dashboard-id (:dashboard-id opts)+ :database-id (:database substituted-query)+ :user-id api/*current-user-id*+ :context (:context opts :action-execute)+ :native? true+ :template (dissoc query :parameters :info)+ :inputs (filterv (comp some? :value) parameters)}+ (fn [result]+ (merge {:result_rows (or (:rows-affected result) 0)} @execution-context))+ (qp/do-with-captured-execution-context #(qp/execute-write-query! substituted-query)+ #(vreset! execution-context %)))))(catch Throwable e(if (= (:type (u/all-ex-data e)) qp.error-type/missing-required-permissions)(api/throw-403 e)@@ -62,8 +80,8 @@{:keys [table-id]} (query/query->database-and-table-ids query)](t2/hydrate (actions.db/table table-id) :fields)))-(defn- execute-custom-action! [action request-parameters]- (let [{action-type :type} action]+(defn- execute-custom-action! [action request-parameters opts]+ (let [{action-type :type, action-id :id} action](actions/check-actions-enabled! action)(let [model (actions.db/card (:model_id action));; the query executes against its own :database; fall back to the derived column if absent@@ -75,10 +93,24 @@(try(case action-type:query- (execute-query-action! action request-parameters)+ (execute-query-action! action request-parameters opts):http- (http-action/execute-http-action! action request-parameters))+ ;; `execute-http-action!` refuses every call today, so this records the refused attempt. The template holds+ ;; `url`/`headers`/`body`, which may carry credentials and `query.query` is plaintext -- so the hash+ ;; identifies the action, never its content.+ (actions.audit/with-audited-execution+ {:action :http/execute+ :action-id action-id+ :dashboard-id (:dashboard-id opts)+ :database-id nil+ :user-id api/*current-user-id*+ :context (:context opts :action-execute)+ :native? false+ :template {:type :internal, :action :http/execute, :action-id action-id}+ :inputs (if (seq request-parameters) [request-parameters] [])}+ (constantly {:result_rows 0})+ (http-action/execute-http-action! action request-parameters)))(catch Exception e(log/errorf "Error executing action: %s" (ex-message e))(if-let [ed (ex-data e)]@@ -175,7 +207,7 @@(legacy->current k k)))(defn- execute-implicit-action!- [action request-parameters]+ [action request-parameters opts](let [model-id (:model_id action)implicit-action (parse-implicit-action action){:keys [query row-parameters]} (build-implicit-query action implicit-action request-parameters)@@ -189,14 +221,17 @@(= implicit-action :model.row/update)(assoc :update-row row-parameters))](binding [qp.perms/*card-id* model-id]- (actions/perform-action! implicit-action arg-map {:scope {:model-id model-id}- :policy :model-action}))))+ (actions/perform-action! implicit-action arg-map {:scope {:model-id model-id}+ :policy :model-action+ :action-id (:id action)+ :dashboard-id (:dashboard-id opts)+ :context (:context opts :action-execute)}))))(mu/defn execute-action!"Execute the given action with the given parameters of shape `{<parameter-id> <value>}."([action request-parameters](execute-action! action request-parameters nil))- ([action request-parameters {:keys [allow-http-actions?] :or {allow-http-actions? true}}]+ ([action request-parameters {:keys [allow-http-actions?] :or {allow-http-actions? true} :as opts}](when (and (= (:type action) :http) (not allow-http-actions?))(throw (ex-info (tru "HTTP actions cannot be executed from public endpoints."){:status-code 403})))@@ -219,9 +254,9 @@request-parameters (merge missing-param-defaults request-parameters)](case (:type action):implicit- (execute-implicit-action! action request-parameters)+ (execute-implicit-action! action request-parameters opts)(:query :http)- (execute-custom-action! action request-parameters)+ (execute-custom-action! action request-parameters opts)(throw (ex-info (tru "Unknown action type {0}." (name (:type action :unknown))) action))))))(mu/defn execute-dashcard!@@ -240,7 +275,7 @@:source :dashboard:type (:type action):action_id (:id action)})- (execute-action! action request-parameters opts))))+ (execute-action! action request-parameters (assoc opts :dashboard-id dashboard-id)))))(defn- fetch-implicit-action-values[action request-parameters]diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljcindex 42a6df5a450..e11f56539f1 100644--- a/src/metabase/lib/schema/info.cljc+++ b/src/metabase/lib/schema/info.cljc@@ -13,7 +13,10 @@[:enum;; do not decode, since this should not get written to the app DB or come in from the REST API.{:decode/normalize identity}+ ;; the prefetch read that pre-fills an implicit action's form -- not the action execution itself,+ ;; which is action-execute (or public-action-execute from the public endpoints):action+ :action-execute:ad-hoc:agent:cache-refresh@@ -27,6 +30,7 @@:csv-download:xlsx-download:json-download+ :public-action-execute:public-dashboard:public-question:public-csv-downloaddiff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.cljindex 1a9b20fc436..58d84f1e611 100644--- a/src/metabase/public_sharing_rest/api.clj+++ b/src/metabase/public_sharing_rest/api.clj@@ -490,7 +490,10 @@(request/as-admin;; Undo middleware string->keyword coercion(actions/execute-dashcard! dashboard-id dashcard-id (update-keys parameters name)- {:allow-http-actions? false})))))))+ ;; `as-admin` grants perms but leaves the user nil, so the audit row has no+ ;; executor; the context is what says the run came from a public link+ {:allow-http-actions? false+ :context :public-action-execute})))))))(defn- iframe"Return an `<iframe>` HTML fragment to embed a public page."@@ -734,7 +737,9 @@:action_id (:id action)});; Undo middleware string->keyword coercion(actions/execute-action! action (update-keys parameters name)- {:allow-http-actions? false})))))))+ ;; see the note on the public dashcard endpoint above+ {:allow-http-actions? false+ :context :public-action-execute})))))));;; ----------------------------------------------------- Map Tiles --------------------------------------------------diff --git a/src/metabase/query_processor/core.clj b/src/metabase/query_processor/core.cljindex ac73f5fb95c..0813bb96f33 100644--- a/src/metabase/query_processor/core.clj+++ b/src/metabase/query_processor/core.clj@@ -15,6 +15,7 @@[metabase.query-processor.middleware.cache.impl :as qp.cache.impl][metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.middleware.limit :as qp.limit]+ [metabase.query-processor.middleware.process-userland-query :as qp.process-userland-query][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.streaming :as qp.streaming]@@ -54,6 +55,9 @@;; Limit — metabase.query-processor.middleware.limit[qp.limitdisable-max-results]+ ;; Userland query — metabase.query-processor.middleware.process-userland-query+ [qp.process-userland-query+ do-with-captured-execution-context];; Pivot — metabase.query-processor.pivot[qp.pivotrun-pivot-query]diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex 90234a617c2..1466c49b62e 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -153,7 +153,9 @@postprocessing middleware (`is_impersonated`, `is_db_routed`, the routed `database_id`) are NOT computed here —they're added later by [[enrich-with-execution-context]] from inside the postprocessing rff, where the bindingsare still in effect. See PR #71386 — reading those values from the query map at the top of the around middleware- was a timing bug because pre-processing hadn't yet run."+ was a timing bug because pre-processing hadn't yet run.++ Mirrored by `execution-row` in `metabase.actions.audit`; keep the two in step."{:arglists '([query])}[{{:keys [executed-by query-hash context action-id card-id dashboard-id transform-id lens-id lens-params pulse-id]:pivot/keys [original-query]} :info@@ -228,6 +230,18 @@(reset! *execution-context-ref* (snapshot-execution-context)))(qp query rff)))+(defn do-with-captured-execution-context+ "Run `f` with [[*execution-context-ref*]] bound, then hand `on-snapshot` whatever+ [[capture-execution-context-middleware]] recorded (nil if it never ran), whether `f` returned or threw. For QP+ entry points that write their own execution row, such as the writeback QP."+ [f on-snapshot]+ (let [context-ref (atom nil)]+ (try+ (binding [*execution-context-ref* context-ref]+ (f))+ (finally+ (on-snapshot @context-ref)))))+(defn- enrich-with-execution-context"Merges the snapshotted execution context (from [[*execution-context-ref*]]) into `execution-info`. Alwaysincludes `:is_impersonated` and `:is_db_routed` (defaulting to `false`) so the QueryExecution row has a stablediff --git a/src/metabase/query_processor/writeback.clj b/src/metabase/query_processor/writeback.cljindex b604c3ee6b4..622377a79fe 100644--- a/src/metabase/query_processor/writeback.clj+++ b/src/metabase/query_processor/writeback.clj@@ -12,6 +12,7 @@[metabase.query-processor.middleware.enterprise :as qp.enterprise][metabase.query-processor.middleware.parameters :as parameters][metabase.query-processor.middleware.permissions :as qp.perms]+ [metabase.query-processor.middleware.process-userland-query :as qp.process-userland-query][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.setup :as qp.setup];; the legacy QP pipeline still conveys the metadata provider via the ambient store; no MBQL 5 path yet@@ -24,7 +25,12 @@"Middleware that happens after compilation, AROUND query execution itself. Has the form(f (f query rff)) -> (f query rff)"- [#'qp.enterprise/swap-destination-db-middleware+ ;; `capture-execution-context-middleware` is first so it runs INNERMOST, inside the `binding`s that the two EE+ ;; middlewares establish -- same ordering rule as [[metabase.query-processor.execute/middleware]]; see its docstring+ ;; for what it captures.+ [#'qp.process-userland-query/capture-execution-context-middleware+ #'qp.enterprise/swap-destination-db-middleware+ #'qp.enterprise/apply-impersonation-postprocessing-middleware#'qp.perms/check-query-action-permissions])(defn- apply-middleware [qp middleware-fns]diff --git a/test/metabase/actions/actions_test.clj b/test/metabase/actions/actions_test.cljindex 86afde72406..1a95c402d51 100644--- a/test/metabase/actions/actions_test.clj+++ b/test/metabase/actions/actions_test.clj@@ -854,3 +854,25 @@(is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))(finally(sql-jdbc.conn/invalidate-pool-for-db! (mt/db)))))))))++(deftest implicit-action-audit-trail-test+ (testing "an implicit action records one QueryExecution row holding an action descriptor, not SQL"+ (with-actions-test-data-and-actions-permissively-enabled!+ (let [since (mt/latest-query-execution-id)+ _ (actions/perform-action! :model.row/create+ (assoc (mt/mbql-query categories)+ :create-row {(format-field-name :name) "audited_row"}))+ rows (mt/action-executions since)]+ (is (= 1 (count rows)))+ (is (=? {:context :action-execute+ :native false+ :action_id nil+ :result_rows 1+ :error nil+ :database_id (mt/id)}+ (first rows)))+ (testing "the query row holds the internal descriptor"+ (is (=? {:type "internal"+ :action "model.row/create"+ :database (mt/id)}+ (t2/select-one-fn :query :model/Query :query_hash (:hash (first rows))))))))))diff --git a/test/metabase/actions/execution_test.clj b/test/metabase/actions/execution_test.cljindex 1535a21cc9d..1f68c7a560e 100644--- a/test/metabase/actions/execution_test.clj+++ b/test/metabase/actions/execution_test.clj@@ -2,10 +2,18 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.actions.execution-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.actions.db :as actions.db][metabase.actions.execution :as actions.execution][metabase.actions.models :as action]+ [metabase.collections.models.collection :as collection]+ [metabase.lib-be.core :as lib-be][metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]- [metabase.test :as mt]))+ [metabase.request.core :as request]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [toucan2.core :as t2])+ (:import+ (clojure.lang ExceptionInfo)))(set! *warn-on-reflection* true)@@ -47,3 +55,219 @@(testing "numeric PK → :number/="(is (= :number/=(:type (first prefetch-parameters)))))))))))++;;; ------------------------------------------------ Audit trail ---------------------------------------------------++(defn- recorded-template [row]+ (t2/select-one-fn :query :model/Query :query_hash (:hash row)))++(deftest native-action-audit-trail-test+ (testing "a native query action records one QueryExecution row for the write"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :query}]+ (mt/with-test-user :crowberto+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)+ result (actions.execution/execute-action! action {"id" 1 "name" "Bird"})+ rows (mt/action-executions since)]+ (is (= {:rows-affected 1} result))+ (is (= 1 (count rows)))+ (is (=? {:context :action-execute+ :action_id action-id+ :executor_id (mt/user->id :crowberto)+ :native true+ :result_rows 1+ :error nil+ :cache_hit false+ :dashboard_id nil+ :database_id (mt/id)}+ (first rows)))+ (testing "the hash covers the template, not the values"+ (is (= (seq (lib-be/query-hash (dissoc (:dataset_query action) :parameters :info)))+ (seq (:hash (first rows))))))+ (testing "the query row holds the template, without the values"+ (let [template (recorded-template (first rows))]+ (is (= (get-in action [:dataset_query :native :query])+ (get-in template [:native :query])))+ (is (not (contains? template :parameters)))))+ (testing "a second execution with different values reuses the query row"+ (let [since (mt/latest-query-execution-id)+ _ (actions.execution/execute-action! action {"id" 1 "name" "Fish"})+ row (first (mt/action-executions since))]+ (is (= (seq (:hash (first rows))) (seq (:hash row))))+ (is (= 1 (t2/count :model/Query :query_hash (:hash row)))))))))))))++(deftest native-action-failure-audit-trail-test+ (testing "a failed native action records the driver's error, and the exception is unchanged"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :query+ :parameters []+ :dataset_query (mt/native-query+ {:query "UPDATE categories SET name = 1/0 WHERE id = 1"})}]+ (mt/with-test-user :crowberto+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)]+ (is (thrown-with-msg? ExceptionInfo #"Error executing Action"+ (actions.execution/execute-action! action {})))+ (let [row (first (mt/action-executions since))]+ (is (some? row))+ (is (some? (:error row)))+ (is (not (re-find #"Error executing Action" (:error row)))+ "the row holds the driver message, not the wrapper added after the audited span")+ (is (= 0 (:result_rows row)))))))))))++(deftest permission-denial-audit-trail-test+ (testing "a permission denial is audited, and still raises a 403"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [_ {:type :model+ :dataset_query (mt/mbql-query categories)+ :collection_id (:id (collection/user->personal-collection (mt/user->id :crowberto)))}+ {query-action-id :action-id} {:type :query}+ {implicit-action-id :action-id} {:type :implicit :kind "row/update"}]+ (mt/with-test-user :rasta+ (testing "native action"+ (let [since (mt/latest-query-execution-id)]+ (is (thrown-with-msg? ExceptionInfo #"permissions"+ (actions.execution/execute-action! (action/select-action :id query-action-id)+ {"id" 1 "name" "Bird"})))+ (is (=? {:context :action-execute, :error some?}+ (first (mt/action-executions since))))))+ ;; the permission check runs inside the audited span, so a denial leaves a row too+ (testing "implicit action"+ (let [since (mt/latest-query-execution-id)]+ (is (thrown-with-msg? ExceptionInfo #"permissions"+ (actions.execution/execute-action! (action/select-action :id implicit-action-id)+ {"id" 1 "name" "Bird"})))+ (is (=? {:context :action-execute, :error some?, :native false}+ (first (mt/action-executions since))))))))))))++(deftest dashcard-and-public-audit-trail-test+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{model-id :id} {:type :model :dataset_query (mt/mbql-query categories)}+ {:keys [action-id]} {:type :query}]+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {}+ :model/DashboardCard {dashcard-id :id} {:dashboard_id dashboard-id+ :action_id action-id+ :card_id model-id}]+ (mt/with-test-user :crowberto+ (testing "a model-page execution leaves dashboard_id nil"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})+ (is (=? {:dashboard_id nil, :context :action-execute}+ (first (mt/action-executions since))))))+ (testing "a dashcard execution sets dashboard_id"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-dashcard! dashboard-id dashcard-id {"id" 1 "name" "Bird"})+ (is (=? {:dashboard_id dashboard-id, :context :action-execute}+ (first (mt/action-executions since))))))+ (testing "a public execution has no executor and its own context"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"}+ {:allow-http-actions? false+ :context :public-action-execute})+ (is (=? {:context :public-action-execute}+ (first (mt/action-executions since))))))+ (testing "a public dashcard execution carries the dashboard and the public context"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-dashcard! dashboard-id dashcard-id {"id" 1 "name" "Bird"}+ {:allow-http-actions? false+ :context :public-action-execute})+ (is (=? {:context :public-action-execute, :dashboard_id dashboard-id}+ (first (mt/action-executions since))))))))))))++(deftest public-execution-has-no-executor-test+ ;; the public endpoints wrap execution in `request/as-admin`, which grants perms but leaves the user nil+ (testing "a public form execution records no executor"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :query}]+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)]+ (request/as-admin+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}+ {:allow-http-actions? false+ :context :public-action-execute}))+ (is (=? {:context :public-action-execute, :executor_id nil}+ (first (mt/action-executions since))))))))))++(deftest public-implicit-execution-skips-impersonation-lookup-test+ ;; the OSS `impersonation-enforced-for-db?` throws without a user, so the audit must not ask it on a public form+ (testing "a public implicit execution records is_impersonated false and logs no lookup error"+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{}+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :implicit :kind "row/update"}]+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)]+ (mt/with-log-messages-for-level [messages [metabase.actions.audit :warn]]+ (request/as-admin+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}+ {:allow-http-actions? false+ :context :public-action-execute}))+ (is (=? {:context :public-action-execute, :executor_id nil, :is_impersonated false}+ (first (mt/action-executions since))))+ (is (empty? (messages)))))))))))++(deftest parameters-are-pii-gated-test+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :query}]+ (mt/with-test-user :crowberto+ (let [action (action/select-action :id action-id)]+ (testing "input values are not recorded by default"+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"})+ (is (=? {:parameterized true, :parameters nil}+ (first (mt/action-executions since))))))+ (testing "input values are recorded once PII retention is on"+ (mt/with-premium-features #{:audit-app}+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]+ (let [since (mt/latest-query-execution-id)]+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"})+ (let [row (first (mt/action-executions since))]+ (is (some? (:parameters row)))+ (is (= #{1 "Bird"}+ (into #{} (map :value) (json/decode+kw (:parameters row))))))))))))))))++(deftest http-action-audit-trail-test+ (testing "a refused HTTP action still leaves a trace, and the trace holds no template"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :http}]+ (mt/with-test-user :crowberto+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)]+ (is (thrown-with-msg? ExceptionInfo #"HTTP actions are disabled."+ (actions.execution/execute-action! action {"id" 1})))+ (let [row (first (mt/action-executions since))]+ (is (=? {:context :action-execute+ :action_id action-id+ :database_id nil+ :native false+ :error "HTTP actions are disabled."}+ row))+ (testing "only the internal descriptor is stored, never url/headers/body"+ (is (= {:type "internal", :action "http/execute", :action-id action-id}+ (recorded-template row))))))))))))++(deftest audit-write-failure-does-not-fail-the-action-test+ (testing "an audit insert that blows up after the warehouse write is logged, not thrown"+ (mt/test-helpers-set-global-values!+ (mt/with-actions-test-data-and-actions-enabled+ (mt/with-actions [{:keys [action-id]} {:type :query}]+ (mt/with-test-user :crowberto+ (let [action (action/select-action :id action-id)+ since (mt/latest-query-execution-id)]+ (let [attempts (atom 0)]+ (mt/with-dynamic-fn-redefs [actions.db/insert-query-execution!+ (fn [_row]+ (swap! attempts inc)+ (throw (ex-info "app db is down" {})))]+ (is (= {:rows-affected 1}+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}))))+ (is (= 1 @attempts)))+ (is (empty? (mt/action-executions since))))))))))diff --git a/test/metabase/actions/test_util.clj b/test/metabase/actions/test_util.cljindex e62cc0e4403..c306e88cff9 100644--- a/test/metabase/actions/test_util.clj+++ b/test/metabase/actions/test_util.clj@@ -347,3 +347,15 @@`(with-actions-test-data(with-actions-enabled~@body)))++(defn latest-query-execution-id+ "The id of the newest QueryExecution row, or 0 when there is none: the `since-id` for [[action-executions]]."+ []+ (or (t2/select-one-pk :model/QueryExecution {:order-by [[:id :desc]]}) 0))++(defn action-executions+ "The action-context QueryExecution rows written after `since-id`, oldest first."+ [since-id]+ (into []+ (filter (comp #{:action-execute :public-action-execute} :context))+ (t2/select :model/QueryExecution {:where [:> :id since-id], :order-by [[:id :asc]]})))diff --git a/test/metabase/queries/models/query_execution_test.clj b/test/metabase/queries/models/query_execution_test.cljnew file mode 100644index 00000000000..1c9f50856dc--- /dev/null+++ b/test/metabase/queries/models/query_execution_test.clj@@ -0,0 +1,28 @@+(ns metabase.queries.models.query-execution-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(defn- query-execution-row [context]+ {:hash (byte-array 32)+ :started_at (t/zoned-date-time)+ :running_time 0+ :result_rows 0+ :native false+ :executor_id (mt/user->id :rasta)+ :context context})++(deftest insert-validates-context-test+ (testing "the action contexts are accepted"+ (doseq [context [:action :action-execute :public-action-execute]]+ (testing context+ (mt/with-model-cleanup [:model/QueryExecution]+ (let [id (t2/insert-returning-pk! :model/QueryExecution (query-execution-row context))]+ (is (= context (t2/select-one-fn :context :model/QueryExecution :id id))))))))+ (testing "an unknown context is still rejected"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Invalid query execution context"+ (t2/insert! :model/QueryExecution (query-execution-row :not-a-real-context))))))diff --git a/test/metabase/test.clj b/test/metabase/test.cljindex e023c9a1827..53abf8e5ff9 100644--- a/test/metabase/test.clj+++ b/test/metabase/test.clj@@ -105,6 +105,8 @@#_{:clj-kondo/ignore [:discouraged-var :deprecated-var]}(p/import-vars[actions.test-util+ action-executions+ latest-query-execution-idwith-actionswith-actions-disabledwith-actions-enabled