The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Show sensitive columns in the sandboxing UI (#66757)
metamben1a2f012afe55
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.cljindex a6f0e3d6d3d..14c3fd5ac40 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj@@ -71,10 +71,12 @@(thunk))))(defenterprise batch-fetch-table-query-metadatas- "Returns the query metadata used to power the Query Builder for the tables specified by`ids`."+ "Returns the query metadata used to power the Query Builder for the tables specified by`ids`.+ Options:+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)":feature :sandboxes- [ids]- (for [table (schema.table/batch-fetch-query-metadatas* ids)]+ [ids opts]+ (for [table (schema.table/batch-fetch-query-metadatas* ids opts)](if (only-sandboxed-perms? table)(filter-fields-for-sandboxingtablediff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.cljindex e4f7e84125d..78af229eae8 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj@@ -110,14 +110,12 @@(testing "Users with restricted access to the columns of a table should only see columns included in the GTAP question"(mt/with-current-user (mt/user->id :rasta)(is (= #{"VENUES.CATEGORY_ID" "VENUES.ID" "VENUES.NAME"}- (->> [(mt/id :venues) (mt/id :checkins)]- table/batch-fetch-table-query-metadatas+ (->> (table/batch-fetch-table-query-metadatas [(mt/id :venues) (mt/id :checkins)] nil)upper-case-field-names)))))(testing "Users with full permissions should not be affected by this field filtering"(mt/with-current-user (mt/user->id :crowberto)(is (= #{"CHECKINS.DATE" "CHECKINS.ID" "CHECKINS.USER_ID" "CHECKINS.VENUE_ID""VENUES.CATEGORY_ID" "VENUES.ID" "VENUES.LATITUDE" "VENUES.LONGITUDE" "VENUES.NAME" "VENUES.PRICE"}- (->> [(mt/id :venues) (mt/id :checkins)]- table/batch-fetch-table-query-metadatas+ (->> (table/batch-fetch-table-query-metadatas [(mt/id :venues) (mt/id :checkins)] nil)upper-case-field-names))))))))diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsxindex 16238b7268c..168b4c4a4e5 100644--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx@@ -218,6 +218,7 @@ const ColumnPicker = ({placeholder={filterByTableColumn ? t`Pick a column` : t`Pick a parameter`}+ includeSensitiveFields/></Box>);diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsxindex 035557ce213..70fa060374a 100644--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx@@ -419,6 +419,7 @@ const TargetName = ({ policy, policyTable, target }: TargetNameProps) => {? getRawDataQuestionForTable(policyTable): null}+ includeSensitiveFields>{({ question }: { question: Question }) => {if (!question) {diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsxindex 4b086706294..26869b7015c 100644--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx@@ -10,7 +10,16 @@ class QuestionParameterTargetWidget extends Component {render() {const { question, ...props } = this.props;const mappingOptions = question- ? getParameterMappingOptions(question, null, question.card())+ ? getParameterMappingOptions(+ question,+ null,+ question.card(),+ null,+ null,+ {+ includeSensitiveFields: true,+ },+ ): [];return (<ParameterTargetWidgetdiff --git a/frontend/src/metabase-lib/filter.ts b/frontend/src/metabase-lib/filter.tsindex 8edf46d2758..e860c0e7f8e 100644--- a/frontend/src/metabase-lib/filter.ts+++ b/frontend/src/metabase-lib/filter.ts@@ -27,11 +27,16 @@ import type {TimeFilterParts,} from "./types";+export type FilterableColumnsOpts = {+ includeSensitiveFields?: boolean;+};+export function filterableColumns(query: Query,stageIndex: number,+ opts?: FilterableColumnsOpts,): ColumnMetadata[] {- return ML.filterable_columns(query, stageIndex);+ return ML.filterable_columns(query, stageIndex, opts);}export function filterableColumnOperators(diff --git a/frontend/src/metabase-lib/v1/parameters/utils/targets.ts b/frontend/src/metabase-lib/v1/parameters/utils/targets.tsindex 2fc32952788..15a96fa3b3c 100644--- a/frontend/src/metabase-lib/v1/parameters/utils/targets.ts+++ b/frontend/src/metabase-lib/v1/parameters/utils/targets.ts@@ -177,7 +177,15 @@ export function buildTextTagTarget(tagName: string): ParameterTextTarget {return ["text-tag", tagName];}-export function getParameterColumns(question: Question, parameter?: Parameter) {+export type GetParameterColumnsOpts = {+ includeSensitiveFields?: boolean;+};++export function getParameterColumns(+ question: Question,+ parameter?: Parameter,+ opts?: GetParameterColumnsOpts,+) {// treat the dataset/model question like it is already composed so that we can apply// dataset/model-specific metadata to the underlying dimension optionsconst query =@@ -199,7 +207,7 @@ export function getParameterColumns(question: Question, parameter?: Parameter) {return { query: nextQuery, columns };}- const availableColumns = getFilterableColumns(nextQuery);+ const availableColumns = getFilterableColumns(nextQuery, opts);const columns = parameter? availableColumns.filter(({ column, stageIndex }) =>columnFilterForParameter(nextQuery, stageIndex, parameter)(column),@@ -222,9 +230,12 @@ function getTemporalColumns(query: Lib.Query, stageIndex: number) {}));}-function getFilterableColumns(query: Lib.Query) {+function getFilterableColumns(+ query: Lib.Query,+ opts?: GetParameterColumnsOpts,+) {return Lib.stageIndexes(query).flatMap((stageIndex) => {- const columns = Lib.filterableColumns(query, stageIndex);+ const columns = Lib.filterableColumns(query, stageIndex, opts);const groups = Lib.groupColumns(columns);return groups.flatMap((group) => {diff --git a/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx b/frontend/src/metabase/common/components/AdHocQuestionLoader.jsxindex 6a4bf0f2a10..446f67aeef1 100644--- a/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx+++ b/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx@@ -97,7 +97,9 @@ export class AdHocQuestionLoader extends Component {// (tables, source db, segments, etc) into// the redux store, the resulting metadata will be available as metadata on the// component props once it's available- await this.props.loadMetadataForCard(card);+ await this.props.loadMetadataForCard(card, {+ includeSensitiveFields: this.props.includeSensitiveFields,+ });// instantiate a new question object using the metadata and saved question// so we can use metabase-lib methods to retrieve information and modify@@ -122,9 +124,11 @@ export class AdHocQuestionLoader extends Component {}// redux stuff-function mapStateToProps(state) {+function mapStateToProps(state, ownProps) {return {- metadata: getMetadata(state),+ metadata: getMetadata(state, {+ includeSensitiveFields: ownProps.includeSensitiveFields,+ }),};}diff --git a/frontend/src/metabase/common/components/QuestionLoader.jsx b/frontend/src/metabase/common/components/QuestionLoader.jsxindex beeb31cb38a..98e0d7b8eb5 100644--- a/frontend/src/metabase/common/components/QuestionLoader.jsx+++ b/frontend/src/metabase/common/components/QuestionLoader.jsx@@ -38,15 +38,22 @@ const QuestionLoader = ({questionObject,questionId,questionHash,+ includeSensitiveFields = false,children,}) =>questionObject != null ? (- <AdHocQuestionLoader questionHash={serializeCardForUrl(questionObject)}>+ <AdHocQuestionLoader+ questionHash={serializeCardForUrl(questionObject)}+ includeSensitiveFields={includeSensitiveFields}+ >{children}</AdHocQuestionLoader>) : // if there's a questionHash it means we're in ad-hoc landquestionHash != null && questionHash !== "" ? (- <AdHocQuestionLoader questionHash={questionHash}>+ <AdHocQuestionLoader+ questionHash={questionHash}+ includeSensitiveFields={includeSensitiveFields}+ >{children}</AdHocQuestionLoader>) : // otherwise if there's a non-null questionId it means we're in saved landdiff --git a/frontend/src/metabase/parameters/utils/mapping-options.ts b/frontend/src/metabase/parameters/utils/mapping-options.tsindex f9368dc67fb..ade75256c5e 100644--- a/frontend/src/metabase/parameters/utils/mapping-options.ts+++ b/frontend/src/metabase/parameters/utils/mapping-options.ts@@ -131,12 +131,17 @@ export type ParameterMappingOption =| StructuredQuerySectionOption| NativeParameterMappingOption;+export type GetParameterMappingOptionsOpts = {+ includeSensitiveFields?: boolean;+};+export function getParameterMappingOptions(question: Question | undefined,parameter: Parameter | null | undefined = null,card: Card | VirtualCard,dashcard: BaseDashboardCard | null | undefined = null,parameterDashcard: BaseDashboardCard | null | undefined = null,+ opts?: GetParameterMappingOptionsOpts,): ParameterMappingOption[] {const isInlineParameterOnCardFromOtherTab =parameterDashcard != null &&@@ -206,6 +211,7 @@ export function getParameterMappingOptions(const { query, columns } = getParameterColumns(question,parameter ?? undefined,+ opts,);const columnsByStageIndex = _.groupBy(columns, "stageIndex");diff --git a/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js b/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.jsindex 9956b5e94a8..ca8107dc466 100644--- a/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js+++ b/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js@@ -606,6 +606,64 @@ describe("parameters/utils/mapping-options", () => {expect(options.length).toBeGreaterThan(0);});});++ describe("includeSensitiveFields option", () => {+ const sensitiveFieldId = 9999;+ const database = createSampleDatabase();+ const ordersTable = createOrdersTable({+ fields: [+ ...createOrdersTable().fields,+ {+ id: sensitiveFieldId,+ table_id: ORDERS_ID,+ name: "SECRET_FIELD",+ display_name: "Secret Field",+ base_type: "type/Text",+ semantic_type: null,+ visibility_type: "sensitive",+ },+ ],+ });+ database.tables = database.tables.map((t) =>+ t.id === ORDERS_ID ? ordersTable : t,+ );++ const metadata = createMockMetadata({ databases: [database] }, undefined, {+ includeSensitiveFields: true,+ });+ const card = structured({ "source-table": ORDERS_ID });+ const question = new Question(card, metadata);++ it("should exclude sensitive fields by default", () => {+ const options = getParameterMappingOptions(+ question,+ { type: "string/=" },+ card,+ );++ const sensitiveOption = options.find(+ (opt) => opt.name === "Secret Field",+ );+ expect(sensitiveOption).toBeUndefined();+ });++ it("should include sensitive fields when includeSensitiveFields is true", () => {+ const options = getParameterMappingOptions(+ question,+ { type: "string/=" },+ card,+ null,+ null,+ { includeSensitiveFields: true },+ );++ const sensitiveOption = options.find(+ (opt) => opt.name === "Secret Field",+ );+ expect(sensitiveOption).toBeDefined();+ expect(sensitiveOption.name).toBe("Secret Field");+ });+ });});describe("getMappingOptionByTarget", () => {diff --git a/frontend/src/metabase/questions/actions.ts b/frontend/src/metabase/questions/actions.tsindex 540a678e007..7cc496ce9ff 100644--- a/frontend/src/metabase/questions/actions.ts+++ b/frontend/src/metabase/questions/actions.ts@@ -16,7 +16,13 @@ export const loadMetadataForTable =};export const loadMetadataForCard =- (card: Card | UnsavedCard, { token }: { token?: EntityToken | null } = {}) =>+ (+ card: Card | UnsavedCard,+ {+ token,+ includeSensitiveFields,+ }: { token?: EntityToken | null; includeSensitiveFields?: boolean } = {},+ ) =>async (dispatch: Dispatch) => {if (isSavedCard(card)) {return entityCompatibleQuery(@@ -30,6 +36,9 @@ export const loadMetadataForCard ={...card.dataset_query,...(!!token && { token }),+ ...(includeSensitiveFields && {+ settings: { include_sensitive_fields: true },+ }),},dispatch,datasetApi.endpoints.getAdhocQueryMetadata,diff --git a/frontend/test/__support__/metadata.ts b/frontend/test/__support__/metadata.tsindex bb9a48cbc17..ed8cf1d963d 100644--- a/frontend/test/__support__/metadata.ts+++ b/frontend/test/__support__/metadata.ts@@ -1,3 +1,4 @@+import type { MetadataSelectorOpts } from "metabase/selectors/metadata";import { getMetadata } from "metabase/selectors/metadata";import type { Settings } from "metabase-types/api";import {@@ -11,11 +12,12 @@ import { createMockEntitiesState } from "./store";export function createMockMetadata(entities: EntitiesStateOpts = {},settings?: Settings,+ metadataOpts?: MetadataSelectorOpts,) {const state = createMockState({entities: createMockEntitiesState(entities),settings: createMockSettingsState(settings),});- return getMetadata(state);+ return getMetadata(state, metadataOpts);}diff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljcindex f3d1d8d59ea..b9cbfe4dbb6 100644--- a/src/metabase/lib/filter.cljc+++ b/src/metabase/lib/filter.cljc@@ -468,10 +468,15 @@([query :- ::lib.schema/querystage-number :- :int]+ (filterable-columns query stage-number nil))++ ([query :- ::lib.schema/query+ stage-number :- :int+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]](let [columns (sequence(comp (map add-column-operators)(clojure.core/filter :operators))- (lib.metadata.calculation/visible-columns query stage-number))+ (lib.metadata.calculation/visible-columns query stage-number options))existing-filters (filters query stage-number)](cond(empty? columns)diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljsindex d9255135b10..e4521600e72 100644--- a/src/metabase/lib/js.cljs+++ b/src/metabase/lib/js.cljs@@ -56,6 +56,7 @@:exclude[filter])(:require+ [clojure.set :as set][clojure.string :as str][goog.object :as gobject][medley.core :as m]@@ -929,15 +930,23 @@The columns have extra information attached, giving the filter operators that can be used with that column.+ `options` is an optional JS object that can include:+ - `includeSensitiveFields`: if true, includes fields with visibility_type :sensitive (default false)+Cached on the query.> **Code health:** Healthy"- [a-query stage-number]- ;; Attaches the cached columns directly to this query, in case it gets called again.- (lib.cache/side-channel-cache- (keyword "filterable-columns" (str "stage-" stage-number)) a-query- (fn [_]- (to-array (lib.core/filterable-columns a-query stage-number)))))+ ([a-query stage-number]+ (filterable-columns a-query stage-number nil))+ ([a-query stage-number options]+ ;; Attaches the cached columns directly to this query, in case it gets called again.+ (let [opts (-> (js-obj->cljs-map options)+ (set/rename-keys {:include-sensitive-fields :include-sensitive-fields?}))+ include-sensitive? (:include-sensitive-fields? opts)]+ (lib.cache/side-channel-cache+ (keyword "filterable-columns" (str "stage-" stage-number (when include-sensitive? "-include-sensitive"))) a-query+ (fn [_]+ (to-array (lib.core/filterable-columns a-query stage-number opts)))))))(defn ^:export filterable-column-operators"Returns the filter operators which can be used in a filter for `filterable-column`.diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljcindex b14d1ccbec1..20ab3ab6121 100644--- a/src/metabase/lib/metadata.cljc+++ b/src/metabase/lib/metadata.cljc@@ -46,29 +46,33 @@table-id :- ::lib.schema.id/table](lib.metadata.protocols/table (->metadata-provider metadata-providerable) table-id))-(defn- fields* [metadata-providerable table-id {:keys [only-active?]}]- (-> (lib.metadata.protocols/fields (->metadata-provider metadata-providerable) table-id)- (cond->> only-active?- (remove (fn [col]- (or (false? (:active col))- (#{:sensitive :retired} (:visibility-type col))))))- (->> (sort-by (juxt #(:position % 0) #(u/lower-case-en (:name % "")))))))+(defn- fields* [metadata-providerable table-id {:keys [include-sensitive?]}]+ (->> (lib.metadata.protocols/fields (->metadata-provider metadata-providerable)+ table-id+ {:include-sensitive? (boolean include-sensitive?)})+ (sort-by (juxt #(:position % 0) #(u/lower-case-en (:name % ""))))))(mu/defn fields :- [:sequential ::lib.schema.metadata/column]"Get metadata about all the Fields belonging to a specific Table."[metadata-providerable :- ::lib.schema.metadata/metadata-providerabletable-id :- ::lib.schema.id/table]- (fields* metadata-providerable table-id {:only-active? false}))+ (fields* metadata-providerable table-id nil))(mu/defn active-fields :- [:sequential ::lib.schema.metadata/column]"Like [[fields]], but filters out any Fields that are not `:active` or with `:visibility-type`s that mean theyshould not be included in queries.These fields are the ones we use for default `:fields`, which becomes the default `SELECT ...` (or equivalent) when- building a query."- [metadata-providerable :- ::lib.schema.metadata/metadata-providerable- table-id :- ::lib.schema.id/table]- (fields* metadata-providerable table-id {:only-active? true}))+ building a query.++ Options:+ - `:include-sensitive?` - if true, includes fields with visibility_type :sensitive (default false)"+ ([metadata-providerable table-id]+ (active-fields metadata-providerable table-id nil))+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ table-id :- ::lib.schema.id/table+ opts]+ (fields* metadata-providerable table-id opts)))(mu/defn metadatas-for-table :- [:maybe [:sequential [:or::lib.schema.metadata/columndiff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljcindex e6e07d3c30f..b63a400e5ee 100644--- a/src/metabase/lib/metadata/calculation.cljc+++ b/src/metabase/lib/metadata/calculation.cljc@@ -493,7 +493,8 @@"Schema for options passed to [[returned-columns]] and [[returned-columns-method]]."[:and[:map- [:include-remaps? {:optional true, :default false} :boolean]]+ [:include-remaps? {:optional true, :default false} :boolean]+ [:include-sensitive-fields? {:optional true, :default false} :boolean]][:fn{:error/message "unique-name-fn is no longer allowed as an option."}(complement :unique-name-fn)]@@ -598,7 +599,8 @@{:include-joined? true:include-expressions? true:include-implicitly-joinable? true- :include-implicitly-joinable-for-source-card? true})+ :include-implicitly-joinable-for-source-card? true+ :include-sensitive-fields? false});;; TODO (Cam 8/7/25) -- historically `visible-columns` worked on a bunch of stuff besides just a stage, but Braden;;; pointed out that it really only makes any sense at all for a stage herediff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljcindex 71c5accba64..72bb76e2546 100644--- a/src/metabase/lib/metadata/protocols.cljc+++ b/src/metabase/lib/metadata/protocols.cljc@@ -19,15 +19,18 @@`:id` and `:name` are mutually exclusive.When fetching metadata that can be inactive/archived/hidden, only active/unarchived/unhidden objects are fetched- unless `:id` or `:name` is specified."+ unless `:id` or `:name` is specifed.++ `:include-sensitive?` can be set to `true` to include Fields with `:visibility-type` `:sensitive` in the results."[:and[:map{:closed true}- [:lib/type [:ref ::metadata-type-excluding-database]]- [:id {:optional true} [:set {:min 1} pos-int?]]- [:name {:optional true} [:set {:min 1} :string]]- [:table-id {:optional true} ::lib.schema.id/table]- [:card-id {:optional true} ::lib.schema.id/card]]+ [:lib/type [:ref ::metadata-type-excluding-database]]+ [:id {:optional true} [:set {:min 1} pos-int?]]+ [:name {:optional true} [:set {:min 1} :string]]+ [:table-id {:optional true} ::lib.schema.id/table]+ [:card-id {:optional true} ::lib.schema.id/card]+ [:include-sensitive? {:optional true} :boolean]][:fn{:error/message ":id and :name cannot be used at the same time."}(complement (every-pred :id :name))]@@ -53,7 +56,7 @@This should match [[metabase.lib-be.metadata.jvm/metadata-spec->honey-sql]] as closely aspossible."- [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec} :- ::metadata-spec]+ [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::metadata-spec](let [active-only? (not (or id-set name-set))metric? (= metadata-type :metadata/metric)preds [(when id-set@@ -76,9 +79,11 @@(not (#{:hidden :technical :cruft} (:visibility-type %)))):metadata/column- #(and- (not (false? (:active %)))- (not (#{:sensitive :retired} (:visibility-type %))))+ (let [excluded-visibility-types (cond-> #{:retired}+ (not include-sensitive?) (conj :sensitive))]+ #(and+ (not (false? (:active %)))+ (not (excluded-visibility-types (:visibility-type %)))))(:metadata/card :metadata/measure :metadata/metric :metadata/segment)#(not (:archived %))@@ -279,10 +284,19 @@(mu/defn fields :- [:maybe [:sequential ::lib.schema.metadata/column]]"Return a sequence of Fields associated with a Table with the given `table-id`. Fields should satisfy- the `:metabase.lib.schema.metadata/column` schema. If no such Table exists, this should error."- [metadata-provider :- ::metadata-provider- table-id :- ::lib.schema.id/table]- (metadatas metadata-provider {:lib/type :metadata/column, :table-id table-id}))+ the `:metabase.lib.schema.metadata/column` schema. If no such Table exists, this should error.++ `opts` is an optional map that can contain:+ - `:include-sensitive?` - if `true`, include Fields with `:visibility-type` `:sensitive` in the results."+ ([metadata-provider :- ::metadata-provider+ table-id :- ::lib.schema.id/table]+ (fields metadata-provider table-id nil))+ ([metadata-provider :- ::metadata-provider+ table-id :- ::lib.schema.id/table+ opts :- [:maybe [:map [:include-sensitive? {:optional true} :boolean]]]]+ (metadatas metadata-provider {:lib/type :metadata/column+ :table-id table-id+ :include-sensitive? (boolean (:include-sensitive? opts))})))(mu/defn segments :- [:maybe [:sequential ::lib.schema.metadata/segment]]"Return a sequence of legacy Segments associated with a Table with the given `table-id`. Segments should satisfydiff --git a/src/metabase/lib/schema/settings.cljc b/src/metabase/lib/schema/settings.cljcindex e93efa5fcbb..5407a06f178 100644--- a/src/metabase/lib/schema/settings.cljc+++ b/src/metabase/lib/schema/settings.cljc@@ -11,4 +11,8 @@[:report-timezone{:optional true:description "The timezone the query should be ran in, overriding the default report timezone for the instance."}- [:ref ::lib.schema.expression.temporal/timezone-id]]])+ [:ref ::lib.schema.expression.temporal/timezone-id]]+ [:include-sensitive-fields+ {:optional true+ :description "Whether to include fields with visibility_type :sensitive when fetching query metadata."}+ :boolean]])diff --git a/src/metabase/lib/table.cljc b/src/metabase/lib/table.cljcindex 59654f4ae29..cbcf57b3f9a 100644--- a/src/metabase/lib/table.cljc+++ b/src/metabase/lib/table.cljc@@ -30,7 +30,7 @@(i18n/tru "Table {0}" (pr-str source-table))))))(mu/defmethod lib.metadata.calculation/returned-columns-method :metadata/table :- ::lib.metadata.calculation/returned-columns- [query _stage-number table-metadata _options]+ [query _stage-number table-metadata {:keys [include-sensitive-fields?]}](into [](comp (map #(assoc % :lib/source :source/table-defaults));; don't truncate column names, if the database says they're ok we should assume they are and we need to@@ -38,4 +38,4 @@;; equal to returned columns for a stage with this source table and nothing else... those SHOULD get;; truncated desired aliases when they stage returned columns are calculated.(lib.field.util/add-source-and-desired-aliases-xform query (lib.util.unique-name-generator/non-truncating-unique-name-generator)))- (lib.metadata/active-fields query (:id table-metadata))))+ (lib.metadata/active-fields query (:id table-metadata) {:include-sensitive? include-sensitive-fields?})))diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex 44b33490705..97695e60d2a 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -472,7 +472,7 @@(case metadata-type:metadata/metric :source_card_id))-(defn- active-only-honeysql-filter [metadata-type]+(defn- active-only-honeysql-filter [metadata-type {:keys [include-sensitive?]}](case metadata-type:metadata/table[:and@@ -482,11 +482,13 @@[:not-in :visibility_type [:inline ["hidden" "technical" "cruft"]]]]]:metadata/column- [:and- [:= :field/active true]- [:or- [:= :field/visibility_type nil]- [:not-in :field/visibility_type [:inline ["sensitive" "retired"]]]]]+ (let [excluded-visibility-types (cond-> ["retired"]+ (not include-sensitive?) (conj "sensitive"))]+ [:and+ [:= :field/active true]+ [:or+ [:= :field/visibility_type nil]+ [:not-in :field/visibility_type [:inline excluded-visibility-types]]]]):metadata/card[:= :card/archived false]@@ -507,8 +509,8 @@{:closed true}[:where {:optional true} vector?]]"This should match [[metabase.lib.metadata.protocols/default-spec-filter-xform]] as closely as possible."- [database-id :- ::lib.schema.id/database- {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]+ [database-id :- ::lib.schema.id/database+ {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec](let [database-id-key (db-id-key metadata-type)active-only? (not (or id-set name-set))metric? (= metadata-type :metadata/metric)@@ -518,7 +520,7 @@name-set (conj [:in (name-key metadata-type) name-set])table-id (conj [:= (table-id-key metadata-type) table-id])card-id (conj [:= (card-id-key metadata-type) card-id])- active-only? (conj (active-only-honeysql-filter metadata-type))+ active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))metric? (conj [:= :type [:inline "metric"]])(and metric? table-id) (conj [:= :source_card_id nil]))](reducediff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.cljindex 7d89e13d873..251dd5ca09c 100644--- a/src/metabase/queries/metadata.clj+++ b/src/metabase/queries/metadata.clj@@ -94,15 +94,18 @@field-id)))(mu/defn- batch-fetch-query-metadata*- "Fetch dependent metadata for ad-hoc queries."- [queries :- [:maybe [:sequential ::lib.schema/query]]]+ "Fetch dependent metadata for ad-hoc queries.+ Options:+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"+ [queries :- [:maybe [:sequential ::lib.schema/query]]+ opts :- [:maybe [:map [:include-sensitive-fields? {:optional true} :boolean]]]](let [source-table-ids (into #{}(mapcat lib/all-source-table-ids)queries)source-card-ids (into #{}(mapcat lib/all-source-card-ids)queries)- source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)+ source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids opts)(schema.table/batch-fetch-card-query-metadatas source-card-ids{:include-database? false}))fk-target-field-ids (into #{} (comp (mapcat :fields)@@ -110,7 +113,7 @@source-tables)fk-target-table-ids (into #{} (remove source-table-ids)(field-ids->table-ids fk-target-field-ids))- fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids)+ fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids opts)tables (concat source-tables fk-target-tables)template-tag-field-ids (into #{} (mapcat lib/all-template-tag-field-ids) queries)direct-snippet-ids (into #{} (mapcat lib/all-template-tag-snippet-ids) queries)@@ -140,13 +143,18 @@:snippets (sort-by :id snippets)}))(defn batch-fetch-query-metadata- "Fetch dependent metadata for ad-hoc queries."- [queries]- (batch-fetch-query-metadata*- (into []- (comp (filter not-empty)- (map lib-be/normalize-query))- queries)))+ "Fetch dependent metadata for ad-hoc queries.+ Options:+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"+ ([queries]+ (batch-fetch-query-metadata queries nil))+ ([queries opts]+ (batch-fetch-query-metadata*+ (into []+ (comp (filter not-empty)+ (map lib-be/normalize-query))+ queries)+ opts)))(mu/defn batch-fetch-card-metadata"Fetch dependent metadata for cards.diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex a5e40eb6e7d..0c0ef46df0d 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -177,13 +177,21 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-route-uses-kebab-case:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/query_metadata"- "Get all of the required query metadata for an ad-hoc query."+ "Get all of the required query metadata for an ad-hoc query.++ You can pass `{:settings {:include-sensitive-fields true}}` in the query to include fields with+ visibility_type :sensitive in the response."[_route-params_query-paramsquery :- [:map- [:database ms/PositiveInt]]]+ [:database ms/PositiveInt]+ [:settings {:optional true} [:maybe [:map+ [:include_sensitive_fields {:optional true} :boolean]]]]]](lib-be/with-metadata-provider-cache- (queries/batch-fetch-query-metadata [query])))+ (queries/batch-fetch-query-metadata+ [query]+ (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]+ {:include-sensitive-fields? include-sensitive-fields}))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happendiff --git a/src/metabase/warehouse_schema/table.clj b/src/metabase/warehouse_schema/table.cljindex be731710352..eabfbeccea6 100644--- a/src/metabase/warehouse_schema/table.clj+++ b/src/metabase/warehouse_schema/table.clj@@ -62,22 +62,28 @@true)))))))(defn batch-fetch-query-metadatas*- "Returns the query metadata used to power the Query Builder for the `table`s specified by `ids`."- [ids]- (when (seq ids)- (let [tables (->> (t2/select :model/Table :id [:in ids])- (filter can-access-table-for-query-metadata?))- tables (t2/hydrate tables- [:fields [:target :has_field_values] :has_field_values :dimensions :name_field]- :segments- :measures- :metrics)]- (for [table tables]- (-> table- (m/dissoc-in [:db :details])- format-fields-for-response- present-table- (update :fields #(remove (comp #{:hidden :sensitive} :visibility_type) %)))))))+ "Returns the query metadata used to power the Query Builder for the `table`s specified by `ids`.+ Options:+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"+ ([ids]+ (batch-fetch-query-metadatas* ids nil))+ ([ids {:keys [include-sensitive-fields?]}]+ (when (seq ids)+ (let [tables (->> (t2/select :model/Table :id [:in ids])+ (filter can-access-table-for-query-metadata?))+ tables (t2/hydrate tables+ [:fields [:target :has_field_values] :has_field_values :dimensions :name_field]+ :segments+ :measures+ :metrics)+ excluded-visibility-types (cond-> #{:hidden}+ (not include-sensitive-fields?) (conj :sensitive))]+ (for [table tables]+ (-> table+ (m/dissoc-in [:db :details])+ format-fields-for-response+ present-table+ (update :fields #(remove (comp excluded-visibility-types :visibility_type) %))))))))(defenterprise fetch-table-query-metadata"Returns the query metadata used to power the Query Builder for the given table `id`. `include-sensitive-fields?`,@@ -87,10 +93,12 @@(fetch-query-metadata* (t2/select-one :model/Table :id id) opts))(defenterprise batch-fetch-table-query-metadatas- "Returns the query metadatas used to power the Query Builder for the tables specified by `ids`."+ "Returns the query metadatas used to power the Query Builder for the tables specified by `ids`.+ Options:+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"metabase-enterprise.sandbox.api.table- [ids]- (batch-fetch-query-metadatas* ids))+ [ids opts]+ (batch-fetch-query-metadatas* ids opts))(defn- card-result-metadata->virtual-fields"Return a sequence of 'virtual' fields metadata for the 'virtual' table for a Card in the Saved Questions 'virtual'diff --git a/test/metabase/lib/metadata/calculation_test.cljc b/test/metabase/lib/metadata/calculation_test.cljcindex 826da8e4495..a3a40ba810b 100644--- a/test/metabase/lib/metadata/calculation_test.cljc+++ b/test/metabase/lib/metadata/calculation_test.cljc@@ -1113,3 +1113,22 @@"Orders → Created At""Orders → Quantity"](map :display-name (lib.metadata.result-metadata/returned-columns query)))))))++(deftest ^:parallel visible-columns-include-sensitive-fields-test+ (testing "visible-columns with :include-sensitive-fields? option"+ (let [mp (lib.tu/merged-mock-metadata-provider+ meta/metadata-provider+ {:fields [{:id (meta/id :venues :latitude)+ :visibility-type :sensitive}+ {:id (meta/id :venues :longitude)+ :visibility-type :retired}]})+ query (lib/query mp (lib.metadata/table mp (meta/id :venues)))]+ (testing "sensitive column is NOT included by default"+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query))]+ (is (not (contains? visible-col-ids (meta/id :venues :latitude))))))+ (testing "sensitive column IS included when :include-sensitive-fields? is true"+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query -1 {:include-sensitive-fields? true}))]+ (is (contains? visible-col-ids (meta/id :venues :latitude)))))+ (testing "retired column is NOT included even with :include-sensitive-fields? true"+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query -1 {:include-sensitive-fields? true}))]+ (is (not (contains? visible-col-ids (meta/id :venues :longitude)))))))))diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex e5a4a5853b3..4e215fabc27 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -1079,3 +1079,20 @@(mt/with-current-user user-id(mt/user-http-request user-id :post 403 "dataset"(mt/mbql-query venues {:limit 1})))))))))))))++(deftest query-metadata-sensitive-fields-test+ (testing "POST /api/dataset/query_metadata"+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:visibility_type :sensitive}+ (testing "sensitive fields are excluded by default"+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/query_metadata"+ (mt/mbql-query venues))]+ (is (not (some #(= (:id %) (mt/id :venues :price))+ (->> result :tables (mapcat :fields))))+ "Sensitive field should NOT be included by default")))+ (testing "sensitive fields are included when :settings :include-sensitive-fields is true"+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/query_metadata"+ (assoc (mt/mbql-query venues)+ :settings {:include_sensitive_fields true}))]+ (is (some #(= (:id %) (mt/id :venues :price))+ (->> result :tables (mapcat :fields)))+ "Sensitive field SHOULD be included when :settings :include-sensitive-fields is true"))))))