Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-3192] audit every action execution in query_execution (#82257)

YB9e27498bbf68
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f6152d616e3..178341a7dd9 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -106,6 +106,7 @@
model-persistence
models
parameters
+ permissions
public-sharing
queries
query-processor
@@ -121,6 +122,7 @@
:model/DashboardCard
:model/Database
:model/Field
+ :model/QueryExecution
:model/Table
:model/User}}
diff --git a/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj b/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj
index 8681564ad93..64ff8773935 100644
--- a/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/action_v2/actions_test.clj
@@ -1,7 +1,12 @@
(ns metabase-enterprise.action-v2.actions-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.action-v2.actions :as action-v2.actions]))
+ [metabase-enterprise.action-v2.actions :as action-v2.actions]
+ [metabase-enterprise.action-v2.test-util :as action-v2.tu]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]))
+
+(use-fixtures :once (fixtures/initialize :test-users-personal-collections))
(def unsupported-dbs-msg #'action-v2.actions/unsupported-dbs-msg)
@@ -10,3 +15,31 @@
(is (= "Data editing isn't supported on one of the target databases." (unsupported-dbs-msg [{:id 1} {:id 2}] [{:id 1}])))
(is (= "Data editing isn't supported on the target databases." (unsupported-dbs-msg [{:id 1} {:id 2}] [{:id 1} {:id 2}])))
(is (= "Data editing isn't supported on some of the target databases." (unsupported-dbs-msg [{:id 1} {:id 2} {:id 3}] [{:id 1} {:id 2}]))))
+
+(deftest data-grid-audit-trail-test
+ (testing "a data-grid write and its undo each record exactly one QueryExecution row"
+ (mt/with-empty-h2-app-db!
+ (mt/with-premium-features #{:table-data-editing}
+ (action-v2.tu/with-test-tables! [table-id [{:id [:int]
+ :name [:text]}
+ {:primary-key [:id]}]]
+ (let [user-id (mt/user->id :crowberto)
+ since (mt/latest-query-execution-id)]
+ (action-v2.tu/create-rows! table-id user-id 200 [{:id 1 :name "Snorkmaiden"}])
+ (let [rows (mt/action-executions since)]
+ (is (= 1 (count rows)))
+ (testing "a data-grid action has no Action row to point at"
+ (is (=? {:context :action-execute
+ :native false
+ :action_id nil
+ :result_rows 1
+ :error nil
+ :database_id (mt/id)}
+ (first rows)))))
+ (testing "undo runs its nested actions under one row"
+ (let [since (mt/latest-query-execution-id)]
+ (mt/user-http-request user-id :post "/ee/action-v2/execute-bulk"
+ {:action :data-editing/undo
+ :scope {:table-id table-id}
+ :inputs [{}]})
+ (is (= 1 (count (mt/action-executions since))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj
index 7e5fbcd23e7..54cfdc4fa54 100644
--- a/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/query_execution_test.clj
@@ -6,6 +6,8 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.impersonation.util-test :as impersonation.util-test]
+ [metabase.actions.execution :as actions.execution]
+ [metabase.actions.models :as action]
[metabase.query-processor :as qp]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.pipeline :as qp.pipeline]
@@ -54,3 +56,48 @@
"QueryExecution row for a FAILED impersonated query should still record is_impersonated=true")
(is (some? (:error (latest-query-execution)))
"Sanity check: the QueryExecution row should have an error message")))))))
+
+(deftest action-row-records-impersonation-test
+ (testing "a native action run under an impersonation policy records is_impersonated, on success and on failure"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{ok-action-id :action-id} {:type :query}
+ {bad-action-id :action-id} {:type :query
+ :parameters []
+ :dataset_query (mt/native-query
+ {:query "UPDATE categories SET name = 1/0 WHERE id = 1"})}]
+ (impersonation.util-test/with-impersonations!
+ {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]
+ :attributes {"impersonation_attr" "impersonation_role"}}
+ (testing "success"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! (action/select-action :id ok-action-id) {"id" 1 "name" "Bird"})
+ (is (=? {:is_impersonated true, :error nil}
+ (first (mt/action-executions since))))))
+ (testing "failure"
+ (let [since (mt/latest-query-execution-id)]
+ (is (thrown? clojure.lang.ExceptionInfo
+ (actions.execution/execute-action! (action/select-action :id bad-action-id) {})))
+ (is (=? {:is_impersonated true, :error some?}
+ (first (mt/action-executions since))))))))))))
+
+(deftest implicit-action-row-records-impersonation-test
+ (testing "an implicit action records is_impersonated from the user's policy on the database, as reads do"
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :implicit :kind "row/update"}]
+ (impersonation.util-test/with-impersonations!
+ {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]
+ :attributes {"impersonation_attr" "impersonation_role"}}
+ ;; H2 can't switch roles, so a true here comes from the policy, as it does for reads on H2
+ (testing "impersonated user"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})
+ (is (=? {:is_impersonated true, :native false, :error nil}
+ (first (mt/action-executions since))))))
+ (testing "admins are never impersonated"
+ (mt/with-test-user :crowberto
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})
+ (is (=? {:is_impersonated false, :error nil}
+ (first (mt/action-executions since)))))))))))))
diff --git a/src/metabase/actions/actions.clj b/src/metabase/actions/actions.clj
index e1515b34f88..49910307588 100644
--- a/src/metabase/actions/actions.clj
+++ b/src/metabase/actions/actions.clj
@@ -3,6 +3,7 @@
(:require
[malli.error :as me]
[metabase.actions.args :as actions.args]
+ [metabase.actions.audit :as actions.audit]
[metabase.actions.db :as actions.db]
[metabase.actions.events :as actions.events]
[metabase.actions.hierarchy :as actions.hierarchy]
@@ -259,7 +260,10 @@
[action
scope
arg-map-or-maps
- & {:keys [policy existing-context user-id]}]
+ ;; `action-id`, `dashboard-id` and `audit-context` are attribution for the audit row; the scope maps are closed
+ ;; schemas, so they ride along as kwargs instead.
+ & {:keys [policy existing-context user-id action-id dashboard-id]
+ audit-context :context}]
(when (and existing-context user-id)
(assert (= user-id (:user-id existing-context)) "Existing context has a consistent user-id"))
(log/with-context {:action action}
@@ -308,21 +312,37 @@
(check-data-editing-enabled-for-database! db))))
(log/with-context {:db-id (:id db)}
(binding [*misc-value-cache* (atom {:databases (zipmap (map :id dbs) dbs)})]
- (check-permissions policy arg-maps)
- (let [result (let [context (-> existing-context
- ;; TODO fix tons of tests which execute without user scope
- (u/assoc-default :user-id (identity #_api/check-500
- (or user-id api/*current-user-id*)))
- (u/assoc-default :scope scope))]
- (if-not driver
- (perform-action-internal! action-kw context arg-maps)
- (driver/with-driver driver
- (let [context (assoc context
- ;; Legacy drivers dispatch on this, for now.
- ;; TODO As far as I'm aware we only have :sql-jdbc defined actions, so can stop dispatching
- ;; on this and just fail if the dynamically determined driver is incompatible.
- :driver driver)]
- (perform-action-internal! action-kw context arg-maps)))))]
+ (let [context (-> existing-context
+ ;; TODO fix tons of tests which execute without user scope
+ (u/assoc-default :user-id (identity #_api/check-500
+ (or user-id api/*current-user-id*)))
+ (u/assoc-default :scope scope))
+ ;; the permission check is inside the audited span so that a denial is traced the same way a
+ ;; driver error is
+ result (actions.audit/with-audited-execution
+ {:action action-kw
+ :action-id action-id
+ :dashboard-id dashboard-id
+ :database-id (:id db)
+ :user-id (:user-id context)
+ :context (or audit-context :action-execute)
+ :native? false
+ :template {:type :internal
+ :action (u/qualified-name action-kw)
+ :database (:id db)
+ :scope (actions.scope/normalize-scope scope)}
+ :inputs arg-maps}
+ (fn [{:keys [outputs]}] {:result_rows (count outputs)})
+ (check-permissions policy arg-maps)
+ (if-not driver
+ (perform-action-internal! action-kw context arg-maps)
+ (driver/with-driver driver
+ (let [context (assoc context
+ ;; Legacy drivers dispatch on this, for now.
+ ;; TODO As far as I'm aware we only have :sql-jdbc defined actions, so can stop dispatching
+ ;; on this and just fail if the dynamically determined driver is incompatible.
+ :driver driver)]
+ (perform-action-internal! action-kw context arg-maps)))))]
{:effects (:effects (:context result))
:outputs (:outputs result)}))))))
diff --git a/src/metabase/actions/audit.clj b/src/metabase/actions/audit.clj
new file mode 100644
index 00000000000..da0eed0b604
--- /dev/null
+++ b/src/metabase/actions/audit.clj
@@ -0,0 +1,113 @@
+(ns metabase.actions.audit
+ "Audit trail for action executions: one QueryExecution row per invocation, written synchronously on the request
+ thread. A failed audit write is logged and never fails the action, so a destructive write goes untraced only when
+ the app DB itself is down.
+
+ The row's `context` (`:action-execute`, or `:public-action-execute` from the public endpoints) separates these
+ write rows from the reads in `v_query_log`; `v_action_log` selects on it. Nested actions are not wrapped, so an
+ invocation is one row."
+ (:require
+ [java-time.api :as t]
+ [metabase.actions.db :as actions.db]
+ [metabase.analytics.settings :as analytics.settings]
+ [metabase.api.common :as api]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.permissions.core :as perms]
+ [metabase.queries.models.query :as query]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private Base
+ "What a recording site knows about an invocation before it runs."
+ [:map {:closed true}
+ ;; e.g. :model.row/update, :query/execute
+ [:action :keyword]
+ [:action-id [:maybe pos-int?]]
+ [:dashboard-id [:maybe pos-int?]]
+ [:database-id [:maybe pos-int?]]
+ [:user-id [:maybe pos-int?]]
+ [:context [:enum :action-execute :public-action-execute]]
+ [:native? :boolean]
+ ;; hashed, and stored in `query` -- the SQL template or an action descriptor, never the input values
+ [:template :map]
+ ;; the input values that were actually supplied: PII-gated into `parameters`
+ [:inputs [:sequential :any]]])
+
+(defn- impersonated?
+ "Whether an enforced connection-impersonation policy applies to the current user on `database-id`: the question the
+ QP's impersonation preprocessing asks, so these rows agree with read rows. A lookup error (e.g. conflicting sandbox
+ and impersonation policies) is logged and reads as false, so auditing never fails the action."
+ [database-id]
+ (boolean
+ ;; public forms run with no user, and the OSS impl throws without one
+ (when (and api/*current-user-id* database-id)
+ (try
+ (perms/impersonation-enforced-for-db? database-id)
+ (catch Throwable e
+ (log/warnf e "Could not resolve connection impersonation for database %d" database-id)
+ false)))))
+
+;; Mirrors `query-execution-info` in [[metabase.query-processor.middleware.process-userland-query]]; keep the two row
+;; shapes in step when a column is added to `query_execution`.
+(mu/defn- execution-row
+ [{:keys [action-id dashboard-id database-id user-id context native? template inputs]} :- Base]
+ {:action_id action-id
+ :dashboard_id dashboard-id
+ :database_id database-id
+ :executor_id user-id
+ :context context
+ :native native?
+ :hash (lib-be/query-hash template)
+ ;; not a column: `save-queries-and-update-average-execution-times!` stores it in `query`, and it is dropped
+ ;; before the insert, as the userland read path does
+ :json_query template
+ :parameterized (boolean (seq inputs))
+ :parameters (when (and (seq inputs) (analytics.settings/analytics-pii-retention-enabled))
+ (json/encode inputs))
+ :tenant_id (:tenant_id @api/*current-user*)
+ :started_at (t/zoned-date-time)
+ ;; an action row can never be a cache hit; setting it keeps the EE cache-rerun join from ever matching
+ :cache_hit false
+ ;; native rows take both from the writeback QP's snapshot; other rows skip the QP, so ask the policy directly
+ :is_impersonated (if native? false (impersonated? database-id))
+ :is_db_routed false
+ :result_rows 0
+ :running_time 0})
+
+(defn do-with-audited-execution
+ "Run `thunk` and record one QueryExecution row for it, whether it succeeded or failed. `base` is a [[Base]] map;
+ `result->row` turns the thunk's return value (nil when it threw) into extra columns, typically `:result_rows`.
+
+ Recording errors are logged, never thrown: the warehouse transaction has already committed or rolled back, and a
+ 500 after a destructive write invites a retry of it. The thunk's own exception is rethrown unchanged."
+ [base result->row thunk]
+ (let [start-ms (System/currentTimeMillis)
+ row (execution-row base)
+ record! (fn [extra-fn]
+ (try
+ (let [row (-> row
+ (merge (extra-fn))
+ (assoc :running_time (u/since-ms-wall-clock start-ms)))]
+ (query/save-queries-and-update-average-execution-times!
+ [{:query (:json_query row)
+ :query-hash (:hash row)
+ :execution-time-ms (:running_time row)}])
+ (actions.db/insert-query-execution! (dissoc row :json_query)))
+ (catch Throwable e
+ (log/errorf e "Failed to record action execution for %s" (:action base)))))]
+ (try
+ (u/prog1 (thunk)
+ (record! #(result->row <>)))
+ (catch Throwable e
+ (record! #(assoc (result->row nil) :error (or (some-> e ex-cause ex-message) (ex-message e))))
+ (throw e)))))
+
+(defmacro with-audited-execution
+ "Macro form of [[do-with-audited-execution]]."
+ {:style/indent 2}
+ [base result->row & body]
+ `(do-with-audited-execution ~base ~result->row (fn [] ~@body)))
diff --git a/src/metabase/actions/db.clj b/src/metabase/actions/db.clj
index 7bb2fc58591..440645bb0c7 100644
--- a/src/metabase/actions/db.clj
+++ b/src/metabase/actions/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.actions.schema :as actions.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -83,6 +84,11 @@
[action-id :- ::lib.schema.id/action]
(t2/delete! :model/DashboardCard :action_id action-id))
+(mu/defn insert-query-execution!
+ "Insert the QueryExecution `row` and return its id."
+ [row :- ::queries.schema/query-execution.update]
+ (t2/insert-returning-pk! :model/QueryExecution row))
+
(mu/defn insert-action!
"Insert the Action `row` and return the inserted instance."
[row :- ::actions.schema/action.for-insert]
diff --git a/src/metabase/actions/execution.clj b/src/metabase/actions/execution.clj
index 0239ed92964..b27fc29f63e 100644
--- a/src/metabase/actions/execution.clj
+++ b/src/metabase/actions/execution.clj
@@ -4,6 +4,7 @@
[medley.core :as m]
[metabase.actions.actions :as actions]
[metabase.actions.args :as actions.args]
+ [metabase.actions.audit :as actions.audit]
[metabase.actions.db :as actions.db]
[metabase.actions.http-action :as http-action]
[metabase.actions.models :as action]
@@ -31,23 +32,40 @@
"Execute a `QueryAction` with parameters as passed in from an
endpoint of shape `{<parameter-id> <value>}`.
- `action` should already be hydrated with its `:card`."
+ `action` should already be hydrated with its `:card`. `opts` carries the audit attribution from the endpoint."
[{query :dataset_query, model-id :model_id, :as action} :- [:map
[:model_id ::lib.schema.id/card]
[:dataset_query ::lib.schema/native-only-query]]
- request-parameters]
+ request-parameters
+ opts]
(log/tracef "Executing action for model %d" model-id)
(driver.conn/with-write-connection
(try
- (let [parameters (for [parameter (:parameters action)]
- ;; the query gets the parameter values, not the frontend's widget settings
- (-> parameter
- (select-keys [:id :type :target :slug :name :default :required :options])
- (assoc :value (get request-parameters (:id parameter)))))
- query (-> query
- (assoc :parameters parameters))]
+ (let [parameters (for [parameter (:parameters action)]
+ ;; the query gets the parameter values, not the frontend's widget settings
+ (-> parameter
+ (select-keys [:id :type :target :slug :name :default :required :options])
+ (assoc :value (get request-parameters (:id parameter)))))
+ substituted-query (-> query
+ (assoc :parameters parameters))
+ ;; the routed destination database and the impersonation flag are only knowable from inside the
+ ;; writeback QP's middleware stack
+ execution-context (volatile! nil)]
(binding [qp.perms/*card-id* model-id]
- (qp/execute-write-query! query)))
+ (actions.audit/with-audited-execution
+ {:action :query/execute
+ :action-id (:id action)
+ :dashboard-id (:dashboard-id opts)
+ :database-id (:database substituted-query)
+ :user-id api/*current-user-id*
+ :context (:context opts :action-execute)
+ :native? true
+ :template (dissoc query :parameters :info)
+ :inputs (filterv (comp some? :value) parameters)}
+ (fn [result]
+ (merge {:result_rows (or (:rows-affected result) 0)} @execution-context))
+ (qp/do-with-captured-execution-context #(qp/execute-write-query! substituted-query)
+ #(vreset! execution-context %)))))
(catch Throwable e
(if (= (:type (u/all-ex-data e)) qp.error-type/missing-required-permissions)
(api/throw-403 e)
@@ -62,8 +80,8 @@
{:keys [table-id]} (query/query->database-and-table-ids query)]
(t2/hydrate (actions.db/table table-id) :fields)))
-(defn- execute-custom-action! [action request-parameters]
- (let [{action-type :type} action]
+(defn- execute-custom-action! [action request-parameters opts]
+ (let [{action-type :type, action-id :id} action]
(actions/check-actions-enabled! action)
(let [model (actions.db/card (:model_id action))
;; the query executes against its own :database; fall back to the derived column if absent
@@ -75,10 +93,24 @@
(try
(case action-type
:query
- (execute-query-action! action request-parameters)
+ (execute-query-action! action request-parameters opts)
:http
- (http-action/execute-http-action! action request-parameters))
+ ;; `execute-http-action!` refuses every call today, so this records the refused attempt. The template holds
+ ;; `url`/`headers`/`body`, which may carry credentials and `query.query` is plaintext -- so the hash
+ ;; identifies the action, never its content.
+ (actions.audit/with-audited-execution
+ {:action :http/execute
+ :action-id action-id
+ :dashboard-id (:dashboard-id opts)
+ :database-id nil
+ :user-id api/*current-user-id*
+ :context (:context opts :action-execute)
+ :native? false
+ :template {:type :internal, :action :http/execute, :action-id action-id}
+ :inputs (if (seq request-parameters) [request-parameters] [])}
+ (constantly {:result_rows 0})
+ (http-action/execute-http-action! action request-parameters)))
(catch Exception e
(log/errorf "Error executing action: %s" (ex-message e))
(if-let [ed (ex-data e)]
@@ -175,7 +207,7 @@
(legacy->current k k)))
(defn- execute-implicit-action!
- [action request-parameters]
+ [action request-parameters opts]
(let [model-id (:model_id action)
implicit-action (parse-implicit-action action)
{:keys [query row-parameters]} (build-implicit-query action implicit-action request-parameters)
@@ -189,14 +221,17 @@
(= implicit-action :model.row/update)
(assoc :update-row row-parameters))]
(binding [qp.perms/*card-id* model-id]
- (actions/perform-action! implicit-action arg-map {:scope {:model-id model-id}
- :policy :model-action}))))
+ (actions/perform-action! implicit-action arg-map {:scope {:model-id model-id}
+ :policy :model-action
+ :action-id (:id action)
+ :dashboard-id (:dashboard-id opts)
+ :context (:context opts :action-execute)}))))
(mu/defn execute-action!
"Execute the given action with the given parameters of shape `{<parameter-id> <value>}."
([action request-parameters]
(execute-action! action request-parameters nil))
- ([action request-parameters {:keys [allow-http-actions?] :or {allow-http-actions? true}}]
+ ([action request-parameters {:keys [allow-http-actions?] :or {allow-http-actions? true} :as opts}]
(when (and (= (:type action) :http) (not allow-http-actions?))
(throw (ex-info (tru "HTTP actions cannot be executed from public endpoints.")
{:status-code 403})))
@@ -219,9 +254,9 @@
request-parameters (merge missing-param-defaults request-parameters)]
(case (:type action)
:implicit
- (execute-implicit-action! action request-parameters)
+ (execute-implicit-action! action request-parameters opts)
(:query :http)
- (execute-custom-action! action request-parameters)
+ (execute-custom-action! action request-parameters opts)
(throw (ex-info (tru "Unknown action type {0}." (name (:type action :unknown))) action))))))
(mu/defn execute-dashcard!
@@ -240,7 +275,7 @@
:source :dashboard
:type (:type action)
:action_id (:id action)})
- (execute-action! action request-parameters opts))))
+ (execute-action! action request-parameters (assoc opts :dashboard-id dashboard-id)))))
(defn- fetch-implicit-action-values
[action request-parameters]
diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljc
index 42a6df5a450..e11f56539f1 100644
--- a/src/metabase/lib/schema/info.cljc
+++ b/src/metabase/lib/schema/info.cljc
@@ -13,7 +13,10 @@
[:enum
;; do not decode, since this should not get written to the app DB or come in from the REST API.
{:decode/normalize identity}
+ ;; the prefetch read that pre-fills an implicit action's form -- not the action execution itself,
+ ;; which is action-execute (or public-action-execute from the public endpoints)
:action
+ :action-execute
:ad-hoc
:agent
:cache-refresh
@@ -27,6 +30,7 @@
:csv-download
:xlsx-download
:json-download
+ :public-action-execute
:public-dashboard
:public-question
:public-csv-download
diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.clj
index 1a9b20fc436..58d84f1e611 100644
--- a/src/metabase/public_sharing_rest/api.clj
+++ b/src/metabase/public_sharing_rest/api.clj
@@ -490,7 +490,10 @@
(request/as-admin
;; Undo middleware string->keyword coercion
(actions/execute-dashcard! dashboard-id dashcard-id (update-keys parameters name)
- {:allow-http-actions? false})))))))
+ ;; `as-admin` grants perms but leaves the user nil, so the audit row has no
+ ;; executor; the context is what says the run came from a public link
+ {:allow-http-actions? false
+ :context :public-action-execute})))))))
(defn- iframe
"Return an `<iframe>` HTML fragment to embed a public page."
@@ -734,7 +737,9 @@
:action_id (:id action)})
;; Undo middleware string->keyword coercion
(actions/execute-action! action (update-keys parameters name)
- {:allow-http-actions? false})))))))
+ ;; see the note on the public dashcard endpoint above
+ {:allow-http-actions? false
+ :context :public-action-execute})))))))
;;; ----------------------------------------------------- Map Tiles --------------------------------------------------
diff --git a/src/metabase/query_processor/core.clj b/src/metabase/query_processor/core.clj
index ac73f5fb95c..0813bb96f33 100644
--- a/src/metabase/query_processor/core.clj
+++ b/src/metabase/query_processor/core.clj
@@ -15,6 +15,7 @@
[metabase.query-processor.middleware.cache.impl :as qp.cache.impl]
[metabase.query-processor.middleware.constraints :as qp.constraints]
[metabase.query-processor.middleware.limit :as qp.limit]
+ [metabase.query-processor.middleware.process-userland-query :as qp.process-userland-query]
[metabase.query-processor.pivot :as qp.pivot]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.streaming :as qp.streaming]
@@ -54,6 +55,9 @@
;; Limit — metabase.query-processor.middleware.limit
[qp.limit
disable-max-results]
+ ;; Userland query — metabase.query-processor.middleware.process-userland-query
+ [qp.process-userland-query
+ do-with-captured-execution-context]
;; Pivot — metabase.query-processor.pivot
[qp.pivot
run-pivot-query]
diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.clj
index 90234a617c2..1466c49b62e 100644
--- a/src/metabase/query_processor/middleware/process_userland_query.clj
+++ b/src/metabase/query_processor/middleware/process_userland_query.clj
@@ -153,7 +153,9 @@
postprocessing middleware (`is_impersonated`, `is_db_routed`, the routed `database_id`) are NOT computed here —
they're added later by [[enrich-with-execution-context]] from inside the postprocessing rff, where the bindings
are still in effect. See PR #71386 — reading those values from the query map at the top of the around middleware
- was a timing bug because pre-processing hadn't yet run."
+ was a timing bug because pre-processing hadn't yet run.
+
+ Mirrored by `execution-row` in `metabase.actions.audit`; keep the two in step."
{:arglists '([query])}
[{{:keys [executed-by query-hash context action-id card-id dashboard-id transform-id lens-id lens-params pulse-id]
:pivot/keys [original-query]} :info
@@ -228,6 +230,18 @@
(reset! *execution-context-ref* (snapshot-execution-context)))
(qp query rff)))
+(defn do-with-captured-execution-context
+ "Run `f` with [[*execution-context-ref*]] bound, then hand `on-snapshot` whatever
+ [[capture-execution-context-middleware]] recorded (nil if it never ran), whether `f` returned or threw. For QP
+ entry points that write their own execution row, such as the writeback QP."
+ [f on-snapshot]
+ (let [context-ref (atom nil)]
+ (try
+ (binding [*execution-context-ref* context-ref]
+ (f))
+ (finally
+ (on-snapshot @context-ref)))))
+
(defn- enrich-with-execution-context
"Merges the snapshotted execution context (from [[*execution-context-ref*]]) into `execution-info`. Always
includes `:is_impersonated` and `:is_db_routed` (defaulting to `false`) so the QueryExecution row has a stable
diff --git a/src/metabase/query_processor/writeback.clj b/src/metabase/query_processor/writeback.clj
index b604c3ee6b4..622377a79fe 100644
--- a/src/metabase/query_processor/writeback.clj
+++ b/src/metabase/query_processor/writeback.clj
@@ -12,6 +12,7 @@
[metabase.query-processor.middleware.enterprise :as qp.enterprise]
[metabase.query-processor.middleware.parameters :as parameters]
[metabase.query-processor.middleware.permissions :as qp.perms]
+ [metabase.query-processor.middleware.process-userland-query :as qp.process-userland-query]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.setup :as qp.setup]
;; the legacy QP pipeline still conveys the metadata provider via the ambient store; no MBQL 5 path yet
@@ -24,7 +25,12 @@
"Middleware that happens after compilation, AROUND query execution itself. Has the form
(f (f query rff)) -> (f query rff)"
- [#'qp.enterprise/swap-destination-db-middleware
+ ;; `capture-execution-context-middleware` is first so it runs INNERMOST, inside the `binding`s that the two EE
+ ;; middlewares establish -- same ordering rule as [[metabase.query-processor.execute/middleware]]; see its docstring
+ ;; for what it captures.
+ [#'qp.process-userland-query/capture-execution-context-middleware
+ #'qp.enterprise/swap-destination-db-middleware
+ #'qp.enterprise/apply-impersonation-postprocessing-middleware
#'qp.perms/check-query-action-permissions])
(defn- apply-middleware [qp middleware-fns]
diff --git a/test/metabase/actions/actions_test.clj b/test/metabase/actions/actions_test.clj
index 86afde72406..1a95c402d51 100644
--- a/test/metabase/actions/actions_test.clj
+++ b/test/metabase/actions/actions_test.clj
@@ -854,3 +854,25 @@
(is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key)))
(finally
(sql-jdbc.conn/invalidate-pool-for-db! (mt/db)))))))))
+
+(deftest implicit-action-audit-trail-test
+ (testing "an implicit action records one QueryExecution row holding an action descriptor, not SQL"
+ (with-actions-test-data-and-actions-permissively-enabled!
+ (let [since (mt/latest-query-execution-id)
+ _ (actions/perform-action! :model.row/create
+ (assoc (mt/mbql-query categories)
+ :create-row {(format-field-name :name) "audited_row"}))
+ rows (mt/action-executions since)]
+ (is (= 1 (count rows)))
+ (is (=? {:context :action-execute
+ :native false
+ :action_id nil
+ :result_rows 1
+ :error nil
+ :database_id (mt/id)}
+ (first rows)))
+ (testing "the query row holds the internal descriptor"
+ (is (=? {:type "internal"
+ :action "model.row/create"
+ :database (mt/id)}
+ (t2/select-one-fn :query :model/Query :query_hash (:hash (first rows))))))))))
diff --git a/test/metabase/actions/execution_test.clj b/test/metabase/actions/execution_test.clj
index 1535a21cc9d..1f68c7a560e 100644
--- a/test/metabase/actions/execution_test.clj
+++ b/test/metabase/actions/execution_test.clj
@@ -2,10 +2,18 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.actions.execution-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.actions.db :as actions.db]
[metabase.actions.execution :as actions.execution]
[metabase.actions.models :as action]
+ [metabase.collections.models.collection :as collection]
+ [metabase.lib-be.core :as lib-be]
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
- [metabase.test :as mt]))
+ [metabase.request.core :as request]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2])
+ (:import
+ (clojure.lang ExceptionInfo)))
(set! *warn-on-reflection* true)
@@ -47,3 +55,219 @@
(testing "numeric PK → :number/="
(is (= :number/=
(:type (first prefetch-parameters)))))))))))
+
+;;; ------------------------------------------------ Audit trail ---------------------------------------------------
+
+(defn- recorded-template [row]
+ (t2/select-one-fn :query :model/Query :query_hash (:hash row)))
+
+(deftest native-action-audit-trail-test
+ (testing "a native query action records one QueryExecution row for the write"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :query}]
+ (mt/with-test-user :crowberto
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)
+ result (actions.execution/execute-action! action {"id" 1 "name" "Bird"})
+ rows (mt/action-executions since)]
+ (is (= {:rows-affected 1} result))
+ (is (= 1 (count rows)))
+ (is (=? {:context :action-execute
+ :action_id action-id
+ :executor_id (mt/user->id :crowberto)
+ :native true
+ :result_rows 1
+ :error nil
+ :cache_hit false
+ :dashboard_id nil
+ :database_id (mt/id)}
+ (first rows)))
+ (testing "the hash covers the template, not the values"
+ (is (= (seq (lib-be/query-hash (dissoc (:dataset_query action) :parameters :info)))
+ (seq (:hash (first rows))))))
+ (testing "the query row holds the template, without the values"
+ (let [template (recorded-template (first rows))]
+ (is (= (get-in action [:dataset_query :native :query])
+ (get-in template [:native :query])))
+ (is (not (contains? template :parameters)))))
+ (testing "a second execution with different values reuses the query row"
+ (let [since (mt/latest-query-execution-id)
+ _ (actions.execution/execute-action! action {"id" 1 "name" "Fish"})
+ row (first (mt/action-executions since))]
+ (is (= (seq (:hash (first rows))) (seq (:hash row))))
+ (is (= 1 (t2/count :model/Query :query_hash (:hash row)))))))))))))
+
+(deftest native-action-failure-audit-trail-test
+ (testing "a failed native action records the driver's error, and the exception is unchanged"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :query
+ :parameters []
+ :dataset_query (mt/native-query
+ {:query "UPDATE categories SET name = 1/0 WHERE id = 1"})}]
+ (mt/with-test-user :crowberto
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)]
+ (is (thrown-with-msg? ExceptionInfo #"Error executing Action"
+ (actions.execution/execute-action! action {})))
+ (let [row (first (mt/action-executions since))]
+ (is (some? row))
+ (is (some? (:error row)))
+ (is (not (re-find #"Error executing Action" (:error row)))
+ "the row holds the driver message, not the wrapper added after the audited span")
+ (is (= 0 (:result_rows row)))))))))))
+
+(deftest permission-denial-audit-trail-test
+ (testing "a permission denial is audited, and still raises a 403"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [_ {:type :model
+ :dataset_query (mt/mbql-query categories)
+ :collection_id (:id (collection/user->personal-collection (mt/user->id :crowberto)))}
+ {query-action-id :action-id} {:type :query}
+ {implicit-action-id :action-id} {:type :implicit :kind "row/update"}]
+ (mt/with-test-user :rasta
+ (testing "native action"
+ (let [since (mt/latest-query-execution-id)]
+ (is (thrown-with-msg? ExceptionInfo #"permissions"
+ (actions.execution/execute-action! (action/select-action :id query-action-id)
+ {"id" 1 "name" "Bird"})))
+ (is (=? {:context :action-execute, :error some?}
+ (first (mt/action-executions since))))))
+ ;; the permission check runs inside the audited span, so a denial leaves a row too
+ (testing "implicit action"
+ (let [since (mt/latest-query-execution-id)]
+ (is (thrown-with-msg? ExceptionInfo #"permissions"
+ (actions.execution/execute-action! (action/select-action :id implicit-action-id)
+ {"id" 1 "name" "Bird"})))
+ (is (=? {:context :action-execute, :error some?, :native false}
+ (first (mt/action-executions since))))))))))))
+
+(deftest dashcard-and-public-audit-trail-test
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{model-id :id} {:type :model :dataset_query (mt/mbql-query categories)}
+ {:keys [action-id]} {:type :query}]
+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {}
+ :model/DashboardCard {dashcard-id :id} {:dashboard_id dashboard-id
+ :action_id action-id
+ :card_id model-id}]
+ (mt/with-test-user :crowberto
+ (testing "a model-page execution leaves dashboard_id nil"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"})
+ (is (=? {:dashboard_id nil, :context :action-execute}
+ (first (mt/action-executions since))))))
+ (testing "a dashcard execution sets dashboard_id"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-dashcard! dashboard-id dashcard-id {"id" 1 "name" "Bird"})
+ (is (=? {:dashboard_id dashboard-id, :context :action-execute}
+ (first (mt/action-executions since))))))
+ (testing "a public execution has no executor and its own context"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! (action/select-action :id action-id) {"id" 1 "name" "Bird"}
+ {:allow-http-actions? false
+ :context :public-action-execute})
+ (is (=? {:context :public-action-execute}
+ (first (mt/action-executions since))))))
+ (testing "a public dashcard execution carries the dashboard and the public context"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-dashcard! dashboard-id dashcard-id {"id" 1 "name" "Bird"}
+ {:allow-http-actions? false
+ :context :public-action-execute})
+ (is (=? {:context :public-action-execute, :dashboard_id dashboard-id}
+ (first (mt/action-executions since))))))))))))
+
+(deftest public-execution-has-no-executor-test
+ ;; the public endpoints wrap execution in `request/as-admin`, which grants perms but leaves the user nil
+ (testing "a public form execution records no executor"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :query}]
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)]
+ (request/as-admin
+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}
+ {:allow-http-actions? false
+ :context :public-action-execute}))
+ (is (=? {:context :public-action-execute, :executor_id nil}
+ (first (mt/action-executions since))))))))))
+
+(deftest public-implicit-execution-skips-impersonation-lookup-test
+ ;; the OSS `impersonation-enforced-for-db?` throws without a user, so the audit must not ask it on a public form
+ (testing "a public implicit execution records is_impersonated false and logs no lookup error"
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{}
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :implicit :kind "row/update"}]
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)]
+ (mt/with-log-messages-for-level [messages [metabase.actions.audit :warn]]
+ (request/as-admin
+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}
+ {:allow-http-actions? false
+ :context :public-action-execute}))
+ (is (=? {:context :public-action-execute, :executor_id nil, :is_impersonated false}
+ (first (mt/action-executions since))))
+ (is (empty? (messages)))))))))))
+
+(deftest parameters-are-pii-gated-test
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :query}]
+ (mt/with-test-user :crowberto
+ (let [action (action/select-action :id action-id)]
+ (testing "input values are not recorded by default"
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"})
+ (is (=? {:parameterized true, :parameters nil}
+ (first (mt/action-executions since))))))
+ (testing "input values are recorded once PII retention is on"
+ (mt/with-premium-features #{:audit-app}
+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]
+ (let [since (mt/latest-query-execution-id)]
+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"})
+ (let [row (first (mt/action-executions since))]
+ (is (some? (:parameters row)))
+ (is (= #{1 "Bird"}
+ (into #{} (map :value) (json/decode+kw (:parameters row))))))))))))))))
+
+(deftest http-action-audit-trail-test
+ (testing "a refused HTTP action still leaves a trace, and the trace holds no template"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :http}]
+ (mt/with-test-user :crowberto
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)]
+ (is (thrown-with-msg? ExceptionInfo #"HTTP actions are disabled."
+ (actions.execution/execute-action! action {"id" 1})))
+ (let [row (first (mt/action-executions since))]
+ (is (=? {:context :action-execute
+ :action_id action-id
+ :database_id nil
+ :native false
+ :error "HTTP actions are disabled."}
+ row))
+ (testing "only the internal descriptor is stored, never url/headers/body"
+ (is (= {:type "internal", :action "http/execute", :action-id action-id}
+ (recorded-template row))))))))))))
+
+(deftest audit-write-failure-does-not-fail-the-action-test
+ (testing "an audit insert that blows up after the warehouse write is logged, not thrown"
+ (mt/test-helpers-set-global-values!
+ (mt/with-actions-test-data-and-actions-enabled
+ (mt/with-actions [{:keys [action-id]} {:type :query}]
+ (mt/with-test-user :crowberto
+ (let [action (action/select-action :id action-id)
+ since (mt/latest-query-execution-id)]
+ (let [attempts (atom 0)]
+ (mt/with-dynamic-fn-redefs [actions.db/insert-query-execution!
+ (fn [_row]
+ (swap! attempts inc)
+ (throw (ex-info "app db is down" {})))]
+ (is (= {:rows-affected 1}
+ (actions.execution/execute-action! action {"id" 1 "name" "Bird"}))))
+ (is (= 1 @attempts)))
+ (is (empty? (mt/action-executions since))))))))))
diff --git a/test/metabase/actions/test_util.clj b/test/metabase/actions/test_util.clj
index e62cc0e4403..c306e88cff9 100644
--- a/test/metabase/actions/test_util.clj
+++ b/test/metabase/actions/test_util.clj
@@ -347,3 +347,15 @@
`(with-actions-test-data
(with-actions-enabled
~@body)))
+
+(defn latest-query-execution-id
+ "The id of the newest QueryExecution row, or 0 when there is none: the `since-id` for [[action-executions]]."
+ []
+ (or (t2/select-one-pk :model/QueryExecution {:order-by [[:id :desc]]}) 0))
+
+(defn action-executions
+ "The action-context QueryExecution rows written after `since-id`, oldest first."
+ [since-id]
+ (into []
+ (filter (comp #{:action-execute :public-action-execute} :context))
+ (t2/select :model/QueryExecution {:where [:> :id since-id], :order-by [[:id :asc]]})))
diff --git a/test/metabase/queries/models/query_execution_test.clj b/test/metabase/queries/models/query_execution_test.clj
new file mode 100644
index 00000000000..1c9f50856dc
--- /dev/null
+++ b/test/metabase/queries/models/query_execution_test.clj
@@ -0,0 +1,28 @@
+(ns metabase.queries.models.query-execution-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(defn- query-execution-row [context]
+ {:hash (byte-array 32)
+ :started_at (t/zoned-date-time)
+ :running_time 0
+ :result_rows 0
+ :native false
+ :executor_id (mt/user->id :rasta)
+ :context context})
+
+(deftest insert-validates-context-test
+ (testing "the action contexts are accepted"
+ (doseq [context [:action :action-execute :public-action-execute]]
+ (testing context
+ (mt/with-model-cleanup [:model/QueryExecution]
+ (let [id (t2/insert-returning-pk! :model/QueryExecution (query-execution-row context))]
+ (is (= context (t2/select-one-fn :context :model/QueryExecution :id id))))))))
+ (testing "an unknown context is still rejected"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid query execution context"
+ (t2/insert! :model/QueryExecution (query-execution-row :not-a-real-context))))))
diff --git a/test/metabase/test.clj b/test/metabase/test.clj
index e023c9a1827..53abf8e5ff9 100644
--- a/test/metabase/test.clj
+++ b/test/metabase/test.clj
@@ -105,6 +105,8 @@
#_{:clj-kondo/ignore [:discouraged-var :deprecated-var]}
(p/import-vars
[actions.test-util
+ action-executions
+ latest-query-execution-id
with-actions
with-actions-disabled
with-actions-enabled