The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Log applied sandboxes in the query execution audit log (#78975)
John Swansone53a1611b4ea
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 801885198c5..e926653f6c4 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -4130,7 +4130,8 @@{:team "UX West":ns-prefix "metabase-enterprise.sandbox":api #{metabase-enterprise.sandbox.api.routes metabase-enterprise.sandbox.api.util metabase-enterprise.sandbox.schema}- :uses #{api+ :uses #{analytics+ apiapi-scopeapp-dbaudit-appdiff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex 4d11a5c33a3..1482af07958 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -8,6 +8,7 @@[metabase-enterprise.sandbox.api.util :as sandbox.api.util][metabase-enterprise.sandbox.db :as sandbox.db][metabase-enterprise.sandbox.models.sandbox :as sandbox]+ [metabase.analytics.settings :as analytics.settings][metabase.api.common :as api :refer [*current-user* *current-user-id*]];; allowed (for now) since sandboxing needs to manipulate legacy metadata^{:clj-kondo/ignore [:discouraged-namespace]}@@ -28,6 +29,7 @@[metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.query-processor.util :as qp.util][metabase.query-processor.util.persisted-cache :as qp.persisted][metabase.request.core :as request][metabase.util :as u]@@ -93,14 +95,19 @@(when (seq sandboxes)(m/index-by :table_id sandboxes))))+(mu/defn- target->field-id :- [:maybe ::lib.schema.id/field]+ "If a parameter `:target` contains a `:field` clause with an integer Field ID, return that ID."+ [target :- ::lib.schema.parameter/target]+ ;; parameter targets still use legacy field refs for whatever wacko reason+ (match/match-one target+ [:field (field-id :guard pos-int?) _opts] field-id))+(mu/defn- target-field->base-type :- [:maybe ::lib.schema.common/base-type]"If the `:target` of a parameter contains a `:field` clause, return the base type corresponding to the Field itreferences. Otherwise returns `nil`."[metadata-providerable :- ::lib.schema.metadata/metadata-providerabletarget-field-clause :- ::lib.schema.parameter/target]- ;; parameter targets still use legacy field refs for whatever wacko reason- (when-let [field-id (match/match-one target-field-clause- [:field (field-id :guard pos-int?) _opts] field-id)]+ (when-let [field-id (target->field-id target-field-clause)](:base-type (lib.metadata/field metadata-providerable field-id))))(defn- attr-value->param-value@@ -398,26 +405,80 @@(log/trace "Applied Sandbox: replaced stage with sandboxed stages")replacement-stages))-(mu/defn- apply-sandboxes :- ::lib.schema/query+(mr/def ::query-with-details+ "A query that has been through [[apply-sandboxing]]. Carries [[metabase.lib.schema/sandboxing.details]] under the+ `::details` key once at least one Sandbox has been applied; the key is absent otherwise.+ [[merge-sandboxing-metadata]] reads it back out during post-processing, and it never leaves the QP: it is an+ internal namespaced key, so [[metabase.lib.serialize/prepare-for-serialization]] drops it before a query is+ returned to the client or saved to the application database."+ [:and+ ::lib.schema/query+ ;; `:malli.core/default` because this conjunct constrains only the one key -- every other key of the query is+ ;; validated by `::lib.schema/query` above, which is where `::details` is declared and closed over+ [:map+ [::details {:optional true} ::lib.schema/sandboxing.details]+ [:malli.core/default :any]]])++(mu/defn- sandbox->details-entry :- ::lib.schema/sandboxing.details.entry+ "Build the [[metabase.lib.schema/sandboxing.details.entry]] for one applied Sandbox. The user's actual attribute+ values are included only when [[analytics.settings/analytics-pii-retention-enabled]] is set, mirroring how that+ setting gates other user-specific data in Usage Analytics."+ [{table-id :table_id+ card-id :card_id+ group-id :group_id+ sandbox-id :id+ remappings :attribute_remappings} :- ::sandbox]+ (let [login-attributes (api/current-user-attributes)+ include-values? (analytics.settings/analytics-pii-retention-enabled)]+ {:table_id table-id+ :sandbox_id sandbox-id+ :group_id group-id+ :card_id card-id+ :attributes (into {}+ (map (fn [[attr-name target]]+ [attr-name (cond-> {:field_id (target->field-id target)}+ include-values? (assoc :value (get login-attributes attr-name)))]))+ remappings)}))++(mu/defn- sandboxed-table-ids :- [:set ::lib.schema.id/table]+ "IDs of the Tables whose Sandboxes have been applied to `query`, read back from the+ `:query-permissions/sandboxed-table` markers that [[apply-sandbox-to-stage]] leaves on each replacement stage. (This+ is the same marker [[merge-sandboxing-metadata]] uses to compute `is_sandboxed`.)"+ [query :- ::lib.schema/query]+ (into #{} (match/match-many query {:query-permissions/sandboxed-table (table-id :guard pos-int?)} table-id)))++(mu/defn- attach-sandbox-details :- ::query-with-details+ "Attach audit-log details under `::details` for the Sandboxes that have been applied to `query`, merged by Table ID since+ sandboxing runs in multiple passes (e.g. again after JOINs are resolved) — markers whose Table has no entry in+ `table-id->sandbox` were applied (and recorded) in an earlier pass. [[merge-sandboxing-metadata]] copies the+ accumulated entries into the results metadata during post-processing."+ [query :- ::query-with-details+ table-id->sandbox :- [:map-of ::lib.schema.id/table ::sandbox]]+ (let [sandboxes (keep table-id->sandbox (sandboxed-table-ids query))]+ (cond-> query+ (seq sandboxes) (update ::details merge (into {} (map (juxt :table_id sandbox->details-entry)) sandboxes)))))++(mu/defn- apply-sandboxes :- ::query-with-details"Replace `:source-table` entries that refer to Tables for which we have applicable Sandboxes with `:source-query`entries from their Sandboxes."[query :- ::lib.schema/querytable-id->sandbox :- [:map-of ::lib.schema.id/table ::sandbox]];; replace stages that have `:source-table` key and a matching entry in `table-id->sandbox`, but do not have;; `::sandbox?` key- (lib.walk/walk-stages- query- (fn [query path stage]- (when (and (= (:lib/type stage) :mbql.stage/mbql)- (:source-table stage)- (not (::sandbox? stage)))- (when-let [sandbox (get table-id->sandbox (:source-table stage))]- ;; add a `::sandbox?` key to each replacement stage that has `:source-table?` so when we do a second pass after- ;; adding JOINs they don't get processed again- (mapv (fn [stage]- (cond-> stage- (:source-table stage) (assoc ::sandbox? true)))- (apply-sandbox-to-stage query path stage sandbox)))))))+ (-> (lib.walk/walk-stages+ query+ (fn [query path stage]+ (when (and (= (:lib/type stage) :mbql.stage/mbql)+ (:source-table stage)+ (not (::sandbox? stage)))+ (when-let [sandbox (get table-id->sandbox (:source-table stage))]+ ;; add a `::sandbox?` key to each replacement stage that has `:source-table?` so when we do a second pass after+ ;; adding JOINs they don't get processed again+ (mapv (fn [stage]+ (cond-> stage+ (:source-table stage) (assoc ::sandbox? true)))+ (apply-sandbox-to-stage query path stage sandbox))))))+ (attach-sandbox-details table-id->sandbox)))(mu/defn- expected-cols :- [:sequential ::mbql.s/legacy-column-metadata][query :- ::lib.schema/query]@@ -432,7 +493,7 @@original-query(assoc sandboxed-query ::original-metadata (expected-cols original-query)))))-(mu/defn- apply-sandboxing* :- ::lib.schema/query+(mu/defn- apply-sandboxing* :- ::query-with-details[query :- ::lib.schema/query](or (when (not api/*is-superuser?*)(when-let [table-id->sandbox (when *current-user-id*@@ -469,10 +530,15 @@(defenterprise merge-sandboxing-metadata"Post-processing middleware. Merges in column metadata from the original, unsandboxed version of the query.":feature :sandboxes- [{::keys [original-metadata] :as query} rff]+ [{::keys [original-metadata details] :as query} rff](fn merge-sandboxing-metadata-rff* [metadata](let [metadata (assoc metadata :is_sandboxed (boolean (match/match-one query{:query-permissions/sandboxed-table &truthy} true)))+ ;; only attach the sandbox audit details for userland queries — they're consumed (and stripped back out of+ ;; the response) by the `process-userland-query` middleware when it records the QueryExecution row+ metadata (cond-> metadata+ (and (seq details) (qp.util/userland-query? query))+ (assoc :sandbox_details (->> (vals details) (sort-by :table_id) vec)))metadata (if original-metadata(merge-metadata original-metadata metadata)metadata)]diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/analytics_dev_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/analytics_dev_test.cljindex 9d6cf29c590..569a6c28924 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/analytics_dev_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/analytics_dev_test.clj@@ -147,7 +147,7 @@"transform_id" "transform_qualified_id""lens_id" "lens_params" "query""embedding_client" "embedding_route" "embedding_sdk_version" "is_preview"- "auth_method" "is_sandboxed" "is_impersonated" "is_db_routed" "parameters"+ "auth_method" "is_sandboxed" "sandbox_details" "is_impersonated" "is_db_routed" "parameters""tenant_id" "embedding_hostname" "embedding_path""user_agent" "sanitized_user_agent" "ip_address""metabase_version"}diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 045b96e0964..9275ce39a02 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -1408,6 +1408,37 @@(is (=? {:is_sandboxed true}(qe))))))))+(deftest sandbox-details-success-test+ (testing "The sandboxes applied to a query are recorded in query_execution.sandbox_details (#71604)"+ (met/with-gtaps! (mt/$ids orders+ {:gtaps {:orders {:remappings {"user_id" [:dimension $user_id]}}}+ :attributes {"user_id" 1}})+ (let [query (mt/mbql-query orders {:aggregation [[:count]]})]+ (testing "the sandbox structure is always recorded, but attribute values are omitted by default"+ (process-userland-query-test/with-query-execution! [qe query]+ (let [result (qp/process-query (qp/userland-query query))]+ (testing "the details are stripped back out of the query results returned to the client"+ (is (not (contains? (:data result) :sandbox_details)))))+ (let [saved-qe (qe)]+ (is (=? {:is_sandboxed true+ :sandbox_details [{:table_id (mt/id :orders)+ :sandbox_id pos-int?+ :group_id pos-int?+ :card_id nil+ :attributes {"user_id" {:field_id (mt/id :orders :user_id)}}}]}+ saved-qe))+ (is (not (contains? (get-in saved-qe [:sandbox_details 0 :attributes "user_id"]) :value))))))+ (testing "the user's attribute values are included when analytics-pii-retention-enabled is set"+ (mt/with-additional-premium-features #{:audit-app}+ (mt/with-temporary-setting-values [analytics-pii-retention-enabled true]+ (process-userland-query-test/with-query-execution! [qe query]+ (qp/process-query (qp/userland-query query))+ ;; login attribute values are stored (and therefore recorded) as strings+ (is (=? {:sandbox_details [{:table_id (mt/id :orders)+ :attributes {"user_id" {:field_id (mt/id :orders :user_id)+ :value "1"}}}]}+ (qe)))))))))))+(deftest sandbox-join-permissions-test(testing "Sandboxed query fails when sandboxed table is joined to a table that the current user doesn't have access to"(met/with-gtaps! (mt/$ids ordersdiff --git a/resources/instance_analytics/collections/main/usage_analytics/query_log.yaml b/resources/instance_analytics/collections/main/usage_analytics/query_log.yamlindex 683ba1cff69..ef7cf6d7cbe 100644--- a/resources/instance_analytics/collections/main/usage_analytics/query_log.yaml+++ b/resources/instance_analytics/collections/main/usage_analytics/query_log.yaml@@ -93,6 +93,9 @@ dataset_query:- - field- {base-type: type/Boolean, effective-type: type/Boolean}- [Internal Metabase Database, public, v_query_log, is_sandboxed]+ - - field+ - {base-type: type/Text, effective-type: type/Text}+ - [Internal Metabase Database, public, v_query_log, sandbox_details]- - field- {base-type: type/Boolean, effective-type: type/Boolean}- [Internal Metabase Database, public, v_query_log, is_impersonated]@@ -515,6 +518,21 @@ result_metadata:semantic_type: type/Categorysettings: nullvisibility_type: normal+- base_type: type/Text+ coercion_strategy: null+ description: null+ display_name: Sandbox Details+ effective_type: type/Text+ field_ref:+ - field+ - [Internal Metabase Database, public, v_query_log, sandbox_details]+ - null+ fk_target_field_id: null+ id: [Internal Metabase Database, public, v_query_log, sandbox_details]+ name: sandbox_details+ semantic_type: type/Category+ settings: null+ visibility_type: normal- base_type: type/Booleancoercion_strategy: nulldescription: null@@ -680,6 +698,7 @@ visualization_settings:- {enabled: true, name: metabase_version}- {enabled: true, name: auth_method}- {enabled: true, name: is_sandboxed}+ - {enabled: true, name: sandbox_details}- {enabled: true, name: is_impersonated}- {enabled: true, name: is_db_routed}- {enabled: true, name: parameters}diff --git a/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sandbox_details.yaml b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sandbox_details.yamlnew file mode 100644index 00000000000..8330c246789--- /dev/null+++ b/resources/instance_analytics/databases/internal_metabase_database/schemas/public/tables/v_query_log/fields/sandbox_details.yaml@@ -0,0 +1,21 @@+name: sandbox_details+display_name: Sandbox Details+created_at: '2026-07-29T00:00:00Z'+visibility_type: normal+table_id: [Internal Metabase Database, public, v_query_log]+database_type: text+base_type: type/Text+effective_type: type/Text+semantic_type: type/Category+position: 40+custom_position: 0+database_position: 40+has_field_values: auto-list+serdes/meta:+- {id: Internal Metabase Database, model: Database}+- {id: public, model: Schema}+- {id: v_query_log, model: Table}+- {id: sandbox_details, model: Field}+database_is_generated: false+database_is_nullable: true+database_is_pk: falsediff --git a/resources/migrations/065/20261006_sandbox_details_audit_log.yaml b/resources/migrations/065/20261006_sandbox_details_audit_log.yamlnew file mode 100644index 00000000000..e75b9be55c4--- /dev/null+++ b/resources/migrations/065/20261006_sandbox_details_audit_log.yaml@@ -0,0 +1,49 @@+## Record which sandboxes (row-level security restrictions) were applied to each query execution, so Usage+## Analytics can show more than an unmodified-looking query for sandboxed runs (UXW-3767).++databaseChangeLog:+ - changeSet:+ id: v65.2026-10-06T10:00:00+ author: johnswanson+ comment: Add query_execution.sandbox_details+ changes:+ - addColumn:+ tableName: query_execution+ columns:+ - column:+ name: sandbox_details+ type: ${text.type}+ remarks: JSON array describing the sandboxes (row-level restrictions) applied to this execution+ constraints:+ nullable: true++ - changeSet:+ id: v65.2026-10-06T10:00:01+ author: johnswanson+ comment: Updated view v_query_log v7 - add sandbox_details (UXW-3767)+ changes:+ - sqlFile:+ dbms: postgresql+ path: ../instance_analytics_views/query_log/v7/postgres-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: ../instance_analytics_views/query_log/v7/mysql-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: ../instance_analytics_views/query_log/v7/h2-query_log.sql+ relativeToChangelogFile: true+ rollback:+ - sqlFile:+ dbms: postgresql+ path: ../instance_analytics_views/query_log/v6/postgres-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: mysql,mariadb+ path: ../instance_analytics_views/query_log/v6/mysql-query_log.sql+ relativeToChangelogFile: true+ - sqlFile:+ dbms: h2+ path: ../instance_analytics_views/query_log/v6/h2-query_log.sql+ relativeToChangelogFile: truediff --git a/resources/migrations/instance_analytics_views/query_log/v7/h2-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v7/h2-query_log.sqlnew file mode 100644index 00000000000..8a7465f69a2--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v7/h2-query_log.sql@@ -0,0 +1,47 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS DOUBLE) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query,+ embedding_client,+ embedding_client_identifier,+ embedding_route,+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,+ embedding_sdk_version,+ metabase_version,+ auth_method,+ is_sandboxed,+ sandbox_details,+ is_impersonated,+ is_db_routed,+ parameters,+ tenant_id,+ embedding_hostname,+ embedding_path,+ user_agent,+ sanitized_user_agent,+ ip_address+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/resources/migrations/instance_analytics_views/query_log/v7/mysql-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v7/mysql-query_log.sqlnew file mode 100644index 00000000000..c6599a716a7--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v7/mysql-query_log.sql@@ -0,0 +1,49 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE+SQL SECURITY INVOKER+VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS DOUBLE) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ concat('card_', card_id) AS card_qualified_id,+ dashboard_id,+ concat('dashboard_', dashboard_id) AS dashboard_qualified_id,+ pulse_id,+ database_id,+ concat('database_', database_id) AS database_qualified_id,+ cache_hit,+ action_id,+ concat('action_', action_id) AS action_qualified_id,+ transform_id,+ concat('transform_', transform_id) AS transform_qualified_id,+ lens_id,+ lens_params,+ query,+ embedding_client,+ embedding_client_identifier,+ embedding_route,+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,+ embedding_sdk_version,+ metabase_version,+ auth_method,+ is_sandboxed,+ sandbox_details,+ is_impersonated,+ is_db_routed,+ parameters,+ tenant_id,+ embedding_hostname,+ embedding_path,+ user_agent,+ sanitized_user_agent,+ ip_address+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/resources/migrations/instance_analytics_views/query_log/v7/postgres-query_log.sql b/resources/migrations/instance_analytics_views/query_log/v7/postgres-query_log.sqlnew file mode 100644index 00000000000..81b70004d9f--- /dev/null+++ b/resources/migrations/instance_analytics_views/query_log/v7/postgres-query_log.sql@@ -0,0 +1,47 @@+DROP VIEW IF EXISTS v_query_log;+++CREATE OR REPLACE VIEW v_query_log AS+SELECT id AS entity_id,+ started_at,+ cast(running_time AS double precision) / 1000 AS running_time_seconds,+ result_rows,+ native AS is_native,+ context AS query_source,+ error,+ coalesce(executor_id, 0) AS user_id,+ card_id,+ 'card_' || card_id AS card_qualified_id,+ dashboard_id,+ 'dashboard_' || dashboard_id AS dashboard_qualified_id,+ pulse_id,+ database_id,+ 'database_' || database_id AS database_qualified_id,+ cache_hit,+ action_id,+ 'action_' || action_id AS action_qualified_id,+ transform_id,+ 'transform_' || transform_id AS transform_qualified_id,+ lens_id,+ lens_params,+ query,+ embedding_client,+ embedding_client_identifier,+ embedding_route,+ CASE WHEN embedding_client LIKE '%-preview' THEN TRUE ELSE FALSE END AS is_preview,+ embedding_sdk_version,+ metabase_version,+ auth_method,+ is_sandboxed,+ sandbox_details,+ is_impersonated,+ is_db_routed,+ parameters,+ tenant_id,+ embedding_hostname,+ embedding_path,+ user_agent,+ sanitized_user_agent,+ ip_address+FROM query_execution+ LEFT JOIN query ON query_execution.hash = query.query_hash;diff --git a/src/metabase/analytics/sdk.clj b/src/metabase/analytics/sdk.cljindex 57939883508..2aad6a73f76 100644--- a/src/metabase/analytics/sdk.clj+++ b/src/metabase/analytics/sdk.clj@@ -17,6 +17,7 @@[metabase.request.user-agent :as request.user-agent][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms])(:import(java.net URI)))@@ -168,7 +169,10 @@[:embedding_route {:optional true} [:maybe :string]][:metabase_version {:optional true} [:maybe :string]][:embedding_client_identifier {:optional true} [:maybe :string]]- [:start_time_millis {:optional true} [:maybe :int]]])+ [:start_time_millis {:optional true} [:maybe :int]]+ ;; the entry shape is enforced where it is built ([[metabase.lib.schema/sandboxing.details.entry]]) and on the way+ ;; into the app DB ([[metabase.queries.schema/query-execution.update]]); this row schema only passes it through+ [:sandbox_details {:optional true} [:maybe [:sequential [:map {::mr/deliberately-open true}]]]]])(mu/defn include-sdk-info :- :map"Adds the currently bound, or existing `*client*` and `*version*` to the given map, which is usually a row goingdiff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.cljindex 87180e941c5..3ef4b9f87dc 100644--- a/src/metabase/analytics/settings.clj+++ b/src/metabase/analytics/settings.clj@@ -111,9 +111,9 @@:setter #'-non-table-chart-generated!)(defsetting analytics-pii-retention-enabled- (deferred-tru (str "Enable logging of embed path, query parameters, user agent, IP address, and Metabot "- "conversation metadata for users of your internal data and embeds. This information "- "will be shown in your usage analytics."))+ (deferred-tru (str "Enable logging of embed path, query parameters, user attribute values, user agent, "+ "IP address, and Metabot conversation metadata for users of your internal data and embeds. "+ "This information will be shown in your usage analytics.")):type :boolean:default false:visibility :admindiff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.cljindex 2bdf137dbaf..5c988b2a0a1 100644--- a/src/metabase/batch_processing/impl.clj+++ b/src/metabase/batch_processing/impl.clj@@ -87,7 +87,10 @@[:metabase_version {:optional true} [:maybe :string]][:embedding_client_identifier {:optional true} [:maybe :string]][:start_time_millis {:optional true} [:maybe :int]]- [:json_query {:optional true} [:maybe query-like]]])+ [:json_query {:optional true} [:maybe query-like]]+ ;; the entry shape is enforced where it is built ([[metabase.lib.schema/sandboxing.details.entry]]) and on the way+ ;; into the app DB ([[metabase.queries.schema/query-execution.update]]); this row schema only passes it through+ [:sandbox_details {:optional true} [:maybe [:sequential [:map {::mr/deliberately-open true}]]]]])(def ^:private id+timestamp[:map {:closed true}diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex a76ea436a72..53a4c12cc16 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -2420,6 +2420,9 @@[:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata{:optional true}:metabase.lib.schema/sandboxing.original-metadata]+ [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/details+ {:optional true}+ :metabase.lib.schema/sandboxing.details];;;; ACTIONS;;diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex afe4fa2bbef..b145f853d5a 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -574,6 +574,23 @@"The legacy columns a query was expected to return before sandboxes replaced its source tables, stashed on the query by the enterprise sandboxing middleware and merged into the sandboxed results."[:sequential [:ref :metabase.legacy-mbql.schema/legacy-column-metadata]])+(mr/def ::sandboxing.details.entry+ "Audit-log entry describing one applied sandbox (row-level restriction): which Table it sandboxes, which sandbox/group/Card it came from, and which login attributes were remapped to which Fields. An attribute's `:value` is the user's actual attribute value, recorded only when `analytics-pii-retention-enabled` is set."+ [:map {:closed true}+ [:table_id [:ref ::id/table]]+ [:sandbox_id [:maybe :int]]+ [:group_id [:maybe :int]]+ [:card_id [:maybe [:ref ::id/card]]]+ [:attributes [:map-of+ #_attribute-name :string+ #_remapping [:map {:closed true}+ [:field_id [:maybe [:ref ::id/field]]]+ [:value {:optional true} [:maybe [:ref ::lib.schema.parameter/parameter.value]]]]]]])++(mr/def ::sandboxing.details+ "Audit-log details for the sandboxes (row-level restrictions) applied to a query, keyed by the Table ID each one sandboxes. Stashed on the query by the enterprise sandboxing middleware and copied into the results metadata so the restrictions can be recorded in the QueryExecution log."+ [:map-of [:ref ::id/table] [:ref ::sandboxing.details.entry]])+(mr/def ::query.snapshot"A whole copy of a query stashed on the query itself under one of the internal keys below: the same shape as[[::query]] minus the `:and`-level constraints, so validating the stashed copy does not re-run uniqueness checks@@ -732,6 +749,9 @@[:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata{:optional true}[:ref ::sandboxing.original-metadata]]+ [:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/details+ {:optional true}+ [:ref ::sandboxing.details]];;;; INFO;;diff --git a/src/metabase/queries/models/query_execution.clj b/src/metabase/queries/models/query_execution.cljindex 35cf05dc180..cf65275dc7e 100644--- a/src/metabase/queries/models/query_execution.clj+++ b/src/metabase/queries/models/query_execution.clj@@ -21,6 +21,8 @@(t2/deftransforms :model/QueryExecution{:json_query mi/transform-json:lens_params mi/transform-json+ ;; no keywordization so login-attribute names round-trip untouched+ :sandbox_details mi/transform-json-no-keywordization:status mi/transform-keyword:context mi/transform-keyword})diff --git a/src/metabase/queries/schema.clj b/src/metabase/queries/schema.cljindex 84c7047659e..f5191899bb9 100644--- a/src/metabase/queries/schema.clj+++ b/src/metabase/queries/schema.clj@@ -4,6 +4,7 @@[metabase.documents.schema :as documents.schema][metabase.lib-be.schema :as lib-be.schema][metabase.lib-metric.schema :as lib-metric.schema]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.parameters.schema :as parameters.schema]@@ -257,6 +258,11 @@[:map {:closed true}[:join_step {:optional true} [:maybe :int]]])+(mr/def ::query-execution.sandbox-details+ "The `:sandbox_details` column of a QueryExecution, decoded: one entry per sandbox (row-level restriction) applied+ to the execution. Built by the enterprise sandboxing middleware."+ [:sequential [:ref ::lib.schema/sandboxing.details.entry]])+(mr/def ::query-execution"A QueryExecution as selected from the app DB: every column of `:query_execution`, plus `:row_count` added by the model's after-select hook."[:merge@@ -302,7 +308,8 @@[:sanitized_user_agent {:optional true} [:maybe :string]][:embedding_route {:optional true} [:maybe :string]][:metabase_version {:optional true} [:maybe :string]]- [:embedding_client_identifier {:optional true} [:maybe :string]]])+ [:embedding_client_identifier {:optional true} [:maybe :string]]+ [:sandbox_details {:optional true} [:maybe ::query-execution.sandbox-details]]])(mr/def ::query-table"A QueryTable as selected from the app DB: every column of `:query_table`."diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex 5fe521a3a19..a2edead9b1a 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -8,6 +8,7 @@(:refer-clojure :exclude [every? empty? get-in not-empty])(:require[java-time.api :as t]+ [medley.core :as m][metabase.analytics-interface.core :as analytics][metabase.analytics.core :as analytics.core][metabase.analytics.settings :as analytics.settings]@@ -99,12 +100,13 @@[](grouper/flush! @save-execution-metadata-queue))-(defn- save-successful-execution-metadata! [cache-details is-sandboxed? query-execution result-rows]+(defn- save-successful-execution-metadata! [cache-details is-sandboxed? sandbox-details query-execution result-rows](let [qe-map (assoc query-execution:cache_hit (boolean (:cached cache-details)):cache_hash (:hash cache-details):result_rows result-rows- :is_sandboxed (boolean is-sandboxed?))]+ :is_sandboxed (boolean is-sandboxed?)+ :sandbox_details sandbox-details)](save-execution-metadata! qe-map)))(defn- save-failed-query-execution! [query-execution message]@@ -122,8 +124,11 @@(-> query-executionadd-running-time(dissoc :error :hash :executor_id :action_id :is_sandboxed :is_impersonated :is_db_routed :card_id :dashboard_id :transform_id :lens_id :lens_params :pulse_id :result_rows :native- :parameterized :parameters))- (dissoc result :cache/details)+ :parameterized :parameters :sandbox_details))+ ;; `[:data :sandbox_details]` exists only to be recorded in the QueryExecution row (see+ ;; [[save-successful-execution-metadata!]]) — it has no business going back to the client+ (-> (dissoc result :cache/details)+ (m/update-existing :data dissoc :sandbox_details)){:cached (when (:cached cache) (:updated_at cache)):status :completed:average_execution_time (when (:cached cache)@@ -144,7 +149,8 @@:card-id (:card_id execution-info):context (:context execution-info)}))(save-successful-execution-metadata!- (:cache/details acc) (get-in acc [:data :is_sandboxed]) execution-info @row-count)+ (:cache/details acc) (get-in acc [:data :is_sandboxed]) (get-in acc [:data :sandbox_details])+ execution-info @row-count)(rf (if (map? acc)(success-response execution-info acc)acc)))diff --git a/src/metabase/query_processor/schema.clj b/src/metabase/query_processor/schema.cljindex 1eab4a43a7c..2dac9612ee3 100644--- a/src/metabase/query_processor/schema.clj+++ b/src/metabase/query_processor/schema.clj@@ -5,6 +5,7 @@^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.schema :as mbql.s][metabase.lib-be.schema :as lib-be.schema]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.util :as u]@@ -92,6 +93,7 @@[false [:fn {:error/message "map"} map?]]]]]]][:pivot? {:optional true} :boolean][:is_sandboxed {:optional true} :boolean]+ [:sandbox_details {:optional true} [:sequential [:ref ::lib.schema/sandboxing.details.entry]]][:download_perms {:optional true} [:or :keyword :string]][:results_metadata {:optional true} [:map {:closed true}[:columns ::result-metadata.columns]]]@@ -166,6 +168,7 @@[:insights {:optional true} [:sequential :map]][:download_perms {:optional true} :string][:is_sandboxed {:optional true} :boolean]+ [:sandbox_details {:optional true} [:sequential :map]][:format-rows? {:optional true} :boolean]])(mr/def ::query-resultdiff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljcindex 1d5edd73f74..1275548bc9f 100644--- a/test/metabase/lib/serialize_test.cljc+++ b/test/metabase/lib/serialize_test.cljc@@ -269,7 +269,13 @@:metabase.query-processor.util.add-alias-info/original clean-query:metabase.query-processor.middleware.add-remaps/external-remaps [external-remap]:metabase-enterprise.sandbox.query-processor.middleware.sandboxing/original-metadata- [{:name "ID", :display_name "ID", :base_type :type/Integer}]})+ [{:name "ID", :display_name "ID", :base_type :type/Integer}]+ :metabase-enterprise.sandbox.query-processor.middleware.sandboxing/details+ {1 {:table_id 1+ :sandbox_id 2+ :group_id 3+ :card_id nil+ :attributes {"user_id" {:field_id 4, :value 5}}}}})(def ^:private dirty-query"A realistic MBQL 5 query carrying every internal key declared on the query, stage, join and options schemas, the waydiff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex f9f23a08a33..8729de9eb0b 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -117,6 +117,7 @@:pulse_id nil:card_id nil:is_sandboxed false+ :sandbox_details nil:dashboard_id nil:transform_id nil:lens_id nildiff --git a/test/metabase/query_processor/middleware/process_userland_query_test.clj b/test/metabase/query_processor/middleware/process_userland_query_test.cljindex 565c87f12a4..b9db966315a 100644--- a/test/metabase/query_processor/middleware/process_userland_query_test.clj+++ b/test/metabase/query_processor/middleware/process_userland_query_test.clj@@ -104,6 +104,7 @@:card_id nil:action_id nil:is_sandboxed false+ :sandbox_details nil:is_impersonated false:is_db_routed false:parameters nil