Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Show sensitive columns in the sandboxing UI (#66757)

metamben1a2f012afe55
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj
index a6f0e3d6d3d..14c3fd5ac40 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj
@@ -71,10 +71,12 @@
(thunk))))
(defenterprise batch-fetch-table-query-metadatas
- "Returns the query metadata used to power the Query Builder for the tables specified by`ids`."
+ "Returns the query metadata used to power the Query Builder for the tables specified by`ids`.
+ Options:
+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"
:feature :sandboxes
- [ids]
- (for [table (schema.table/batch-fetch-query-metadatas* ids)]
+ [ids opts]
+ (for [table (schema.table/batch-fetch-query-metadatas* ids opts)]
(if (only-sandboxed-perms? table)
(filter-fields-for-sandboxing
table
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
index e4f7e84125d..78af229eae8 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
@@ -110,14 +110,12 @@
(testing "Users with restricted access to the columns of a table should only see columns included in the GTAP question"
(mt/with-current-user (mt/user->id :rasta)
(is (= #{"VENUES.CATEGORY_ID" "VENUES.ID" "VENUES.NAME"}
- (->> [(mt/id :venues) (mt/id :checkins)]
- table/batch-fetch-table-query-metadatas
+ (->> (table/batch-fetch-table-query-metadatas [(mt/id :venues) (mt/id :checkins)] nil)
upper-case-field-names)))))
(testing "Users with full permissions should not be affected by this field filtering"
(mt/with-current-user (mt/user->id :crowberto)
(is (= #{"CHECKINS.DATE" "CHECKINS.ID" "CHECKINS.USER_ID" "CHECKINS.VENUE_ID"
"VENUES.CATEGORY_ID" "VENUES.ID" "VENUES.LATITUDE" "VENUES.LONGITUDE" "VENUES.NAME" "VENUES.PRICE"}
- (->> [(mt/id :venues) (mt/id :checkins)]
- table/batch-fetch-table-query-metadatas
+ (->> (table/batch-fetch-table-query-metadatas [(mt/id :venues) (mt/id :checkins)] nil)
upper-case-field-names))))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx
index 16238b7268c..168b4c4a4e5 100644
--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/AttributeMappingEditor/DataAttributeMappingEditor.tsx
@@ -218,6 +218,7 @@ const ColumnPicker = ({
placeholder={
filterByTableColumn ? t`Pick a column` : t`Pick a parameter`
}
+ includeSensitiveFields
/>
</Box>
);
diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx
index 035557ce213..70fa060374a 100644
--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/components/EditSandboxingModal/EditSandboxingModal.tsx
@@ -419,6 +419,7 @@ const TargetName = ({ policy, policyTable, target }: TargetNameProps) => {
? getRawDataQuestionForTable(policyTable)
: null
}
+ includeSensitiveFields
>
{({ question }: { question: Question }) => {
if (!question) {
diff --git a/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx b/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx
index 4b086706294..26869b7015c 100644
--- a/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx
+++ b/enterprise/frontend/src/metabase-enterprise/sandboxes/containers/QuestionParameterTargetWidget.jsx
@@ -10,7 +10,16 @@ class QuestionParameterTargetWidget extends Component {
render() {
const { question, ...props } = this.props;
const mappingOptions = question
- ? getParameterMappingOptions(question, null, question.card())
+ ? getParameterMappingOptions(
+ question,
+ null,
+ question.card(),
+ null,
+ null,
+ {
+ includeSensitiveFields: true,
+ },
+ )
: [];
return (
<ParameterTargetWidget
diff --git a/frontend/src/metabase-lib/filter.ts b/frontend/src/metabase-lib/filter.ts
index 8edf46d2758..e860c0e7f8e 100644
--- a/frontend/src/metabase-lib/filter.ts
+++ b/frontend/src/metabase-lib/filter.ts
@@ -27,11 +27,16 @@ import type {
TimeFilterParts,
} from "./types";
+export type FilterableColumnsOpts = {
+ includeSensitiveFields?: boolean;
+};
+
export function filterableColumns(
query: Query,
stageIndex: number,
+ opts?: FilterableColumnsOpts,
): ColumnMetadata[] {
- return ML.filterable_columns(query, stageIndex);
+ return ML.filterable_columns(query, stageIndex, opts);
}
export function filterableColumnOperators(
diff --git a/frontend/src/metabase-lib/v1/parameters/utils/targets.ts b/frontend/src/metabase-lib/v1/parameters/utils/targets.ts
index 2fc32952788..15a96fa3b3c 100644
--- a/frontend/src/metabase-lib/v1/parameters/utils/targets.ts
+++ b/frontend/src/metabase-lib/v1/parameters/utils/targets.ts
@@ -177,7 +177,15 @@ export function buildTextTagTarget(tagName: string): ParameterTextTarget {
return ["text-tag", tagName];
}
-export function getParameterColumns(question: Question, parameter?: Parameter) {
+export type GetParameterColumnsOpts = {
+ includeSensitiveFields?: boolean;
+};
+
+export function getParameterColumns(
+ question: Question,
+ parameter?: Parameter,
+ opts?: GetParameterColumnsOpts,
+) {
// treat the dataset/model question like it is already composed so that we can apply
// dataset/model-specific metadata to the underlying dimension options
const query =
@@ -199,7 +207,7 @@ export function getParameterColumns(question: Question, parameter?: Parameter) {
return { query: nextQuery, columns };
}
- const availableColumns = getFilterableColumns(nextQuery);
+ const availableColumns = getFilterableColumns(nextQuery, opts);
const columns = parameter
? availableColumns.filter(({ column, stageIndex }) =>
columnFilterForParameter(nextQuery, stageIndex, parameter)(column),
@@ -222,9 +230,12 @@ function getTemporalColumns(query: Lib.Query, stageIndex: number) {
}));
}
-function getFilterableColumns(query: Lib.Query) {
+function getFilterableColumns(
+ query: Lib.Query,
+ opts?: GetParameterColumnsOpts,
+) {
return Lib.stageIndexes(query).flatMap((stageIndex) => {
- const columns = Lib.filterableColumns(query, stageIndex);
+ const columns = Lib.filterableColumns(query, stageIndex, opts);
const groups = Lib.groupColumns(columns);
return groups.flatMap((group) => {
diff --git a/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx b/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx
index 6a4bf0f2a10..446f67aeef1 100644
--- a/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx
+++ b/frontend/src/metabase/common/components/AdHocQuestionLoader.jsx
@@ -97,7 +97,9 @@ export class AdHocQuestionLoader extends Component {
// (tables, source db, segments, etc) into
// the redux store, the resulting metadata will be available as metadata on the
// component props once it's available
- await this.props.loadMetadataForCard(card);
+ await this.props.loadMetadataForCard(card, {
+ includeSensitiveFields: this.props.includeSensitiveFields,
+ });
// instantiate a new question object using the metadata and saved question
// so we can use metabase-lib methods to retrieve information and modify
@@ -122,9 +124,11 @@ export class AdHocQuestionLoader extends Component {
}
// redux stuff
-function mapStateToProps(state) {
+function mapStateToProps(state, ownProps) {
return {
- metadata: getMetadata(state),
+ metadata: getMetadata(state, {
+ includeSensitiveFields: ownProps.includeSensitiveFields,
+ }),
};
}
diff --git a/frontend/src/metabase/common/components/QuestionLoader.jsx b/frontend/src/metabase/common/components/QuestionLoader.jsx
index beeb31cb38a..98e0d7b8eb5 100644
--- a/frontend/src/metabase/common/components/QuestionLoader.jsx
+++ b/frontend/src/metabase/common/components/QuestionLoader.jsx
@@ -38,15 +38,22 @@ const QuestionLoader = ({
questionObject,
questionId,
questionHash,
+ includeSensitiveFields = false,
children,
}) =>
questionObject != null ? (
- <AdHocQuestionLoader questionHash={serializeCardForUrl(questionObject)}>
+ <AdHocQuestionLoader
+ questionHash={serializeCardForUrl(questionObject)}
+ includeSensitiveFields={includeSensitiveFields}
+ >
{children}
</AdHocQuestionLoader>
) : // if there's a questionHash it means we're in ad-hoc land
questionHash != null && questionHash !== "" ? (
- <AdHocQuestionLoader questionHash={questionHash}>
+ <AdHocQuestionLoader
+ questionHash={questionHash}
+ includeSensitiveFields={includeSensitiveFields}
+ >
{children}
</AdHocQuestionLoader>
) : // otherwise if there's a non-null questionId it means we're in saved land
diff --git a/frontend/src/metabase/parameters/utils/mapping-options.ts b/frontend/src/metabase/parameters/utils/mapping-options.ts
index f9368dc67fb..ade75256c5e 100644
--- a/frontend/src/metabase/parameters/utils/mapping-options.ts
+++ b/frontend/src/metabase/parameters/utils/mapping-options.ts
@@ -131,12 +131,17 @@ export type ParameterMappingOption =
| StructuredQuerySectionOption
| NativeParameterMappingOption;
+export type GetParameterMappingOptionsOpts = {
+ includeSensitiveFields?: boolean;
+};
+
export function getParameterMappingOptions(
question: Question | undefined,
parameter: Parameter | null | undefined = null,
card: Card | VirtualCard,
dashcard: BaseDashboardCard | null | undefined = null,
parameterDashcard: BaseDashboardCard | null | undefined = null,
+ opts?: GetParameterMappingOptionsOpts,
): ParameterMappingOption[] {
const isInlineParameterOnCardFromOtherTab =
parameterDashcard != null &&
@@ -206,6 +211,7 @@ export function getParameterMappingOptions(
const { query, columns } = getParameterColumns(
question,
parameter ?? undefined,
+ opts,
);
const columnsByStageIndex = _.groupBy(columns, "stageIndex");
diff --git a/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js b/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js
index 9956b5e94a8..ca8107dc466 100644
--- a/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js
+++ b/frontend/src/metabase/parameters/utils/mapping-options.unit.spec.js
@@ -606,6 +606,64 @@ describe("parameters/utils/mapping-options", () => {
expect(options.length).toBeGreaterThan(0);
});
});
+
+ describe("includeSensitiveFields option", () => {
+ const sensitiveFieldId = 9999;
+ const database = createSampleDatabase();
+ const ordersTable = createOrdersTable({
+ fields: [
+ ...createOrdersTable().fields,
+ {
+ id: sensitiveFieldId,
+ table_id: ORDERS_ID,
+ name: "SECRET_FIELD",
+ display_name: "Secret Field",
+ base_type: "type/Text",
+ semantic_type: null,
+ visibility_type: "sensitive",
+ },
+ ],
+ });
+ database.tables = database.tables.map((t) =>
+ t.id === ORDERS_ID ? ordersTable : t,
+ );
+
+ const metadata = createMockMetadata({ databases: [database] }, undefined, {
+ includeSensitiveFields: true,
+ });
+ const card = structured({ "source-table": ORDERS_ID });
+ const question = new Question(card, metadata);
+
+ it("should exclude sensitive fields by default", () => {
+ const options = getParameterMappingOptions(
+ question,
+ { type: "string/=" },
+ card,
+ );
+
+ const sensitiveOption = options.find(
+ (opt) => opt.name === "Secret Field",
+ );
+ expect(sensitiveOption).toBeUndefined();
+ });
+
+ it("should include sensitive fields when includeSensitiveFields is true", () => {
+ const options = getParameterMappingOptions(
+ question,
+ { type: "string/=" },
+ card,
+ null,
+ null,
+ { includeSensitiveFields: true },
+ );
+
+ const sensitiveOption = options.find(
+ (opt) => opt.name === "Secret Field",
+ );
+ expect(sensitiveOption).toBeDefined();
+ expect(sensitiveOption.name).toBe("Secret Field");
+ });
+ });
});
describe("getMappingOptionByTarget", () => {
diff --git a/frontend/src/metabase/questions/actions.ts b/frontend/src/metabase/questions/actions.ts
index 540a678e007..7cc496ce9ff 100644
--- a/frontend/src/metabase/questions/actions.ts
+++ b/frontend/src/metabase/questions/actions.ts
@@ -16,7 +16,13 @@ export const loadMetadataForTable =
};
export const loadMetadataForCard =
- (card: Card | UnsavedCard, { token }: { token?: EntityToken | null } = {}) =>
+ (
+ card: Card | UnsavedCard,
+ {
+ token,
+ includeSensitiveFields,
+ }: { token?: EntityToken | null; includeSensitiveFields?: boolean } = {},
+ ) =>
async (dispatch: Dispatch) => {
if (isSavedCard(card)) {
return entityCompatibleQuery(
@@ -30,6 +36,9 @@ export const loadMetadataForCard =
{
...card.dataset_query,
...(!!token && { token }),
+ ...(includeSensitiveFields && {
+ settings: { include_sensitive_fields: true },
+ }),
},
dispatch,
datasetApi.endpoints.getAdhocQueryMetadata,
diff --git a/frontend/test/__support__/metadata.ts b/frontend/test/__support__/metadata.ts
index bb9a48cbc17..ed8cf1d963d 100644
--- a/frontend/test/__support__/metadata.ts
+++ b/frontend/test/__support__/metadata.ts
@@ -1,3 +1,4 @@
+import type { MetadataSelectorOpts } from "metabase/selectors/metadata";
import { getMetadata } from "metabase/selectors/metadata";
import type { Settings } from "metabase-types/api";
import {
@@ -11,11 +12,12 @@ import { createMockEntitiesState } from "./store";
export function createMockMetadata(
entities: EntitiesStateOpts = {},
settings?: Settings,
+ metadataOpts?: MetadataSelectorOpts,
) {
const state = createMockState({
entities: createMockEntitiesState(entities),
settings: createMockSettingsState(settings),
});
- return getMetadata(state);
+ return getMetadata(state, metadataOpts);
}
diff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljc
index f3d1d8d59ea..b9cbfe4dbb6 100644
--- a/src/metabase/lib/filter.cljc
+++ b/src/metabase/lib/filter.cljc
@@ -468,10 +468,15 @@
([query :- ::lib.schema/query
stage-number :- :int]
+ (filterable-columns query stage-number nil))
+
+ ([query :- ::lib.schema/query
+ stage-number :- :int
+ options :- [:maybe ::lib.metadata.calculation/visible-columns.options]]
(let [columns (sequence
(comp (map add-column-operators)
(clojure.core/filter :operators))
- (lib.metadata.calculation/visible-columns query stage-number))
+ (lib.metadata.calculation/visible-columns query stage-number options))
existing-filters (filters query stage-number)]
(cond
(empty? columns)
diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljs
index d9255135b10..e4521600e72 100644
--- a/src/metabase/lib/js.cljs
+++ b/src/metabase/lib/js.cljs
@@ -56,6 +56,7 @@
:exclude
[filter])
(:require
+ [clojure.set :as set]
[clojure.string :as str]
[goog.object :as gobject]
[medley.core :as m]
@@ -929,15 +930,23 @@
The columns have extra information attached, giving the filter operators that can be used with that column.
+ `options` is an optional JS object that can include:
+ - `includeSensitiveFields`: if true, includes fields with visibility_type :sensitive (default false)
+
Cached on the query.
> **Code health:** Healthy"
- [a-query stage-number]
- ;; Attaches the cached columns directly to this query, in case it gets called again.
- (lib.cache/side-channel-cache
- (keyword "filterable-columns" (str "stage-" stage-number)) a-query
- (fn [_]
- (to-array (lib.core/filterable-columns a-query stage-number)))))
+ ([a-query stage-number]
+ (filterable-columns a-query stage-number nil))
+ ([a-query stage-number options]
+ ;; Attaches the cached columns directly to this query, in case it gets called again.
+ (let [opts (-> (js-obj->cljs-map options)
+ (set/rename-keys {:include-sensitive-fields :include-sensitive-fields?}))
+ include-sensitive? (:include-sensitive-fields? opts)]
+ (lib.cache/side-channel-cache
+ (keyword "filterable-columns" (str "stage-" stage-number (when include-sensitive? "-include-sensitive"))) a-query
+ (fn [_]
+ (to-array (lib.core/filterable-columns a-query stage-number opts)))))))
(defn ^:export filterable-column-operators
"Returns the filter operators which can be used in a filter for `filterable-column`.
diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljc
index b14d1ccbec1..20ab3ab6121 100644
--- a/src/metabase/lib/metadata.cljc
+++ b/src/metabase/lib/metadata.cljc
@@ -46,29 +46,33 @@
table-id :- ::lib.schema.id/table]
(lib.metadata.protocols/table (->metadata-provider metadata-providerable) table-id))
-(defn- fields* [metadata-providerable table-id {:keys [only-active?]}]
- (-> (lib.metadata.protocols/fields (->metadata-provider metadata-providerable) table-id)
- (cond->> only-active?
- (remove (fn [col]
- (or (false? (:active col))
- (#{:sensitive :retired} (:visibility-type col))))))
- (->> (sort-by (juxt #(:position % 0) #(u/lower-case-en (:name % "")))))))
+(defn- fields* [metadata-providerable table-id {:keys [include-sensitive?]}]
+ (->> (lib.metadata.protocols/fields (->metadata-provider metadata-providerable)
+ table-id
+ {:include-sensitive? (boolean include-sensitive?)})
+ (sort-by (juxt #(:position % 0) #(u/lower-case-en (:name % ""))))))
(mu/defn fields :- [:sequential ::lib.schema.metadata/column]
"Get metadata about all the Fields belonging to a specific Table."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
table-id :- ::lib.schema.id/table]
- (fields* metadata-providerable table-id {:only-active? false}))
+ (fields* metadata-providerable table-id nil))
(mu/defn active-fields :- [:sequential ::lib.schema.metadata/column]
"Like [[fields]], but filters out any Fields that are not `:active` or with `:visibility-type`s that mean they
should not be included in queries.
These fields are the ones we use for default `:fields`, which becomes the default `SELECT ...` (or equivalent) when
- building a query."
- [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- table-id :- ::lib.schema.id/table]
- (fields* metadata-providerable table-id {:only-active? true}))
+ building a query.
+
+ Options:
+ - `:include-sensitive?` - if true, includes fields with visibility_type :sensitive (default false)"
+ ([metadata-providerable table-id]
+ (active-fields metadata-providerable table-id nil))
+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ table-id :- ::lib.schema.id/table
+ opts]
+ (fields* metadata-providerable table-id opts)))
(mu/defn metadatas-for-table :- [:maybe [:sequential [:or
::lib.schema.metadata/column
diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljc
index e6e07d3c30f..b63a400e5ee 100644
--- a/src/metabase/lib/metadata/calculation.cljc
+++ b/src/metabase/lib/metadata/calculation.cljc
@@ -493,7 +493,8 @@
"Schema for options passed to [[returned-columns]] and [[returned-columns-method]]."
[:and
[:map
- [:include-remaps? {:optional true, :default false} :boolean]]
+ [:include-remaps? {:optional true, :default false} :boolean]
+ [:include-sensitive-fields? {:optional true, :default false} :boolean]]
[:fn
{:error/message "unique-name-fn is no longer allowed as an option."}
(complement :unique-name-fn)]
@@ -598,7 +599,8 @@
{:include-joined? true
:include-expressions? true
:include-implicitly-joinable? true
- :include-implicitly-joinable-for-source-card? true})
+ :include-implicitly-joinable-for-source-card? true
+ :include-sensitive-fields? false})
;;; TODO (Cam 8/7/25) -- historically `visible-columns` worked on a bunch of stuff besides just a stage, but Braden
;;; pointed out that it really only makes any sense at all for a stage here
diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljc
index 71c5accba64..72bb76e2546 100644
--- a/src/metabase/lib/metadata/protocols.cljc
+++ b/src/metabase/lib/metadata/protocols.cljc
@@ -19,15 +19,18 @@
`:id` and `:name` are mutually exclusive.
When fetching metadata that can be inactive/archived/hidden, only active/unarchived/unhidden objects are fetched
- unless `:id` or `:name` is specified."
+ unless `:id` or `:name` is specifed.
+
+ `:include-sensitive?` can be set to `true` to include Fields with `:visibility-type` `:sensitive` in the results."
[:and
[:map
{:closed true}
- [:lib/type [:ref ::metadata-type-excluding-database]]
- [:id {:optional true} [:set {:min 1} pos-int?]]
- [:name {:optional true} [:set {:min 1} :string]]
- [:table-id {:optional true} ::lib.schema.id/table]
- [:card-id {:optional true} ::lib.schema.id/card]]
+ [:lib/type [:ref ::metadata-type-excluding-database]]
+ [:id {:optional true} [:set {:min 1} pos-int?]]
+ [:name {:optional true} [:set {:min 1} :string]]
+ [:table-id {:optional true} ::lib.schema.id/table]
+ [:card-id {:optional true} ::lib.schema.id/card]
+ [:include-sensitive? {:optional true} :boolean]]
[:fn
{:error/message ":id and :name cannot be used at the same time."}
(complement (every-pred :id :name))]
@@ -53,7 +56,7 @@
This should match [[metabase.lib-be.metadata.jvm/metadata-spec->honey-sql]] as closely as
possible."
- [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec} :- ::metadata-spec]
+ [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::metadata-spec]
(let [active-only? (not (or id-set name-set))
metric? (= metadata-type :metadata/metric)
preds [(when id-set
@@ -76,9 +79,11 @@
(not (#{:hidden :technical :cruft} (:visibility-type %))))
:metadata/column
- #(and
- (not (false? (:active %)))
- (not (#{:sensitive :retired} (:visibility-type %))))
+ (let [excluded-visibility-types (cond-> #{:retired}
+ (not include-sensitive?) (conj :sensitive))]
+ #(and
+ (not (false? (:active %)))
+ (not (excluded-visibility-types (:visibility-type %)))))
(:metadata/card :metadata/measure :metadata/metric :metadata/segment)
#(not (:archived %))
@@ -279,10 +284,19 @@
(mu/defn fields :- [:maybe [:sequential ::lib.schema.metadata/column]]
"Return a sequence of Fields associated with a Table with the given `table-id`. Fields should satisfy
- the `:metabase.lib.schema.metadata/column` schema. If no such Table exists, this should error."
- [metadata-provider :- ::metadata-provider
- table-id :- ::lib.schema.id/table]
- (metadatas metadata-provider {:lib/type :metadata/column, :table-id table-id}))
+ the `:metabase.lib.schema.metadata/column` schema. If no such Table exists, this should error.
+
+ `opts` is an optional map that can contain:
+ - `:include-sensitive?` - if `true`, include Fields with `:visibility-type` `:sensitive` in the results."
+ ([metadata-provider :- ::metadata-provider
+ table-id :- ::lib.schema.id/table]
+ (fields metadata-provider table-id nil))
+ ([metadata-provider :- ::metadata-provider
+ table-id :- ::lib.schema.id/table
+ opts :- [:maybe [:map [:include-sensitive? {:optional true} :boolean]]]]
+ (metadatas metadata-provider {:lib/type :metadata/column
+ :table-id table-id
+ :include-sensitive? (boolean (:include-sensitive? opts))})))
(mu/defn segments :- [:maybe [:sequential ::lib.schema.metadata/segment]]
"Return a sequence of legacy Segments associated with a Table with the given `table-id`. Segments should satisfy
diff --git a/src/metabase/lib/schema/settings.cljc b/src/metabase/lib/schema/settings.cljc
index e93efa5fcbb..5407a06f178 100644
--- a/src/metabase/lib/schema/settings.cljc
+++ b/src/metabase/lib/schema/settings.cljc
@@ -11,4 +11,8 @@
[:report-timezone
{:optional true
:description "The timezone the query should be ran in, overriding the default report timezone for the instance."}
- [:ref ::lib.schema.expression.temporal/timezone-id]]])
+ [:ref ::lib.schema.expression.temporal/timezone-id]]
+ [:include-sensitive-fields
+ {:optional true
+ :description "Whether to include fields with visibility_type :sensitive when fetching query metadata."}
+ :boolean]])
diff --git a/src/metabase/lib/table.cljc b/src/metabase/lib/table.cljc
index 59654f4ae29..cbcf57b3f9a 100644
--- a/src/metabase/lib/table.cljc
+++ b/src/metabase/lib/table.cljc
@@ -30,7 +30,7 @@
(i18n/tru "Table {0}" (pr-str source-table))))))
(mu/defmethod lib.metadata.calculation/returned-columns-method :metadata/table :- ::lib.metadata.calculation/returned-columns
- [query _stage-number table-metadata _options]
+ [query _stage-number table-metadata {:keys [include-sensitive-fields?]}]
(into []
(comp (map #(assoc % :lib/source :source/table-defaults))
;; don't truncate column names, if the database says they're ok we should assume they are and we need to
@@ -38,4 +38,4 @@
;; equal to returned columns for a stage with this source table and nothing else... those SHOULD get
;; truncated desired aliases when they stage returned columns are calculated.
(lib.field.util/add-source-and-desired-aliases-xform query (lib.util.unique-name-generator/non-truncating-unique-name-generator)))
- (lib.metadata/active-fields query (:id table-metadata))))
+ (lib.metadata/active-fields query (:id table-metadata) {:include-sensitive? include-sensitive-fields?})))
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 44b33490705..97695e60d2a 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -472,7 +472,7 @@
(case metadata-type
:metadata/metric :source_card_id))
-(defn- active-only-honeysql-filter [metadata-type]
+(defn- active-only-honeysql-filter [metadata-type {:keys [include-sensitive?]}]
(case metadata-type
:metadata/table
[:and
@@ -482,11 +482,13 @@
[:not-in :visibility_type [:inline ["hidden" "technical" "cruft"]]]]]
:metadata/column
- [:and
- [:= :field/active true]
- [:or
- [:= :field/visibility_type nil]
- [:not-in :field/visibility_type [:inline ["sensitive" "retired"]]]]]
+ (let [excluded-visibility-types (cond-> ["retired"]
+ (not include-sensitive?) (conj "sensitive"))]
+ [:and
+ [:= :field/active true]
+ [:or
+ [:= :field/visibility_type nil]
+ [:not-in :field/visibility_type [:inline excluded-visibility-types]]]])
:metadata/card
[:= :card/archived false]
@@ -507,8 +509,8 @@
{:closed true}
[:where {:optional true} vector?]]
"This should match [[metabase.lib.metadata.protocols/default-spec-filter-xform]] as closely as possible."
- [database-id :- ::lib.schema.id/database
- {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
+ [database-id :- ::lib.schema.id/database
+ {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
(let [database-id-key (db-id-key metadata-type)
active-only? (not (or id-set name-set))
metric? (= metadata-type :metadata/metric)
@@ -518,7 +520,7 @@
name-set (conj [:in (name-key metadata-type) name-set])
table-id (conj [:= (table-id-key metadata-type) table-id])
card-id (conj [:= (card-id-key metadata-type) card-id])
- active-only? (conj (active-only-honeysql-filter metadata-type))
+ active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))
metric? (conj [:= :type [:inline "metric"]])
(and metric? table-id) (conj [:= :source_card_id nil]))]
(reduce
diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.clj
index 7d89e13d873..251dd5ca09c 100644
--- a/src/metabase/queries/metadata.clj
+++ b/src/metabase/queries/metadata.clj
@@ -94,15 +94,18 @@
field-id)))
(mu/defn- batch-fetch-query-metadata*
- "Fetch dependent metadata for ad-hoc queries."
- [queries :- [:maybe [:sequential ::lib.schema/query]]]
+ "Fetch dependent metadata for ad-hoc queries.
+ Options:
+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"
+ [queries :- [:maybe [:sequential ::lib.schema/query]]
+ opts :- [:maybe [:map [:include-sensitive-fields? {:optional true} :boolean]]]]
(let [source-table-ids (into #{}
(mapcat lib/all-source-table-ids)
queries)
source-card-ids (into #{}
(mapcat lib/all-source-card-ids)
queries)
- source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)
+ source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids opts)
(schema.table/batch-fetch-card-query-metadatas source-card-ids
{:include-database? false}))
fk-target-field-ids (into #{} (comp (mapcat :fields)
@@ -110,7 +113,7 @@
source-tables)
fk-target-table-ids (into #{} (remove source-table-ids)
(field-ids->table-ids fk-target-field-ids))
- fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids)
+ fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids opts)
tables (concat source-tables fk-target-tables)
template-tag-field-ids (into #{} (mapcat lib/all-template-tag-field-ids) queries)
direct-snippet-ids (into #{} (mapcat lib/all-template-tag-snippet-ids) queries)
@@ -140,13 +143,18 @@
:snippets (sort-by :id snippets)}))
(defn batch-fetch-query-metadata
- "Fetch dependent metadata for ad-hoc queries."
- [queries]
- (batch-fetch-query-metadata*
- (into []
- (comp (filter not-empty)
- (map lib-be/normalize-query))
- queries)))
+ "Fetch dependent metadata for ad-hoc queries.
+ Options:
+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"
+ ([queries]
+ (batch-fetch-query-metadata queries nil))
+ ([queries opts]
+ (batch-fetch-query-metadata*
+ (into []
+ (comp (filter not-empty)
+ (map lib-be/normalize-query))
+ queries)
+ opts)))
(mu/defn batch-fetch-card-metadata
"Fetch dependent metadata for cards.
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index a5e40eb6e7d..0c0ef46df0d 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -177,13 +177,21 @@
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-route-uses-kebab-case
:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/query_metadata"
- "Get all of the required query metadata for an ad-hoc query."
+ "Get all of the required query metadata for an ad-hoc query.
+
+ You can pass `{:settings {:include-sensitive-fields true}}` in the query to include fields with
+ visibility_type :sensitive in the response."
[_route-params
_query-params
query :- [:map
- [:database ms/PositiveInt]]]
+ [:database ms/PositiveInt]
+ [:settings {:optional true} [:maybe [:map
+ [:include_sensitive_fields {:optional true} :boolean]]]]]]
(lib-be/with-metadata-provider-cache
- (queries/batch-fetch-query-metadata [query])))
+ (queries/batch-fetch-query-metadata
+ [query]
+ (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]
+ {:include-sensitive-fields? include-sensitive-fields}))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
diff --git a/src/metabase/warehouse_schema/table.clj b/src/metabase/warehouse_schema/table.clj
index be731710352..eabfbeccea6 100644
--- a/src/metabase/warehouse_schema/table.clj
+++ b/src/metabase/warehouse_schema/table.clj
@@ -62,22 +62,28 @@
true)))))))
(defn batch-fetch-query-metadatas*
- "Returns the query metadata used to power the Query Builder for the `table`s specified by `ids`."
- [ids]
- (when (seq ids)
- (let [tables (->> (t2/select :model/Table :id [:in ids])
- (filter can-access-table-for-query-metadata?))
- tables (t2/hydrate tables
- [:fields [:target :has_field_values] :has_field_values :dimensions :name_field]
- :segments
- :measures
- :metrics)]
- (for [table tables]
- (-> table
- (m/dissoc-in [:db :details])
- format-fields-for-response
- present-table
- (update :fields #(remove (comp #{:hidden :sensitive} :visibility_type) %)))))))
+ "Returns the query metadata used to power the Query Builder for the `table`s specified by `ids`.
+ Options:
+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"
+ ([ids]
+ (batch-fetch-query-metadatas* ids nil))
+ ([ids {:keys [include-sensitive-fields?]}]
+ (when (seq ids)
+ (let [tables (->> (t2/select :model/Table :id [:in ids])
+ (filter can-access-table-for-query-metadata?))
+ tables (t2/hydrate tables
+ [:fields [:target :has_field_values] :has_field_values :dimensions :name_field]
+ :segments
+ :measures
+ :metrics)
+ excluded-visibility-types (cond-> #{:hidden}
+ (not include-sensitive-fields?) (conj :sensitive))]
+ (for [table tables]
+ (-> table
+ (m/dissoc-in [:db :details])
+ format-fields-for-response
+ present-table
+ (update :fields #(remove (comp excluded-visibility-types :visibility_type) %))))))))
(defenterprise fetch-table-query-metadata
"Returns the query metadata used to power the Query Builder for the given table `id`. `include-sensitive-fields?`,
@@ -87,10 +93,12 @@
(fetch-query-metadata* (t2/select-one :model/Table :id id) opts))
(defenterprise batch-fetch-table-query-metadatas
- "Returns the query metadatas used to power the Query Builder for the tables specified by `ids`."
+ "Returns the query metadatas used to power the Query Builder for the tables specified by `ids`.
+ Options:
+ - `include-sensitive-fields?` - if true, includes fields with visibility_type :sensitive (default false)"
metabase-enterprise.sandbox.api.table
- [ids]
- (batch-fetch-query-metadatas* ids))
+ [ids opts]
+ (batch-fetch-query-metadatas* ids opts))
(defn- card-result-metadata->virtual-fields
"Return a sequence of 'virtual' fields metadata for the 'virtual' table for a Card in the Saved Questions 'virtual'
diff --git a/test/metabase/lib/metadata/calculation_test.cljc b/test/metabase/lib/metadata/calculation_test.cljc
index 826da8e4495..a3a40ba810b 100644
--- a/test/metabase/lib/metadata/calculation_test.cljc
+++ b/test/metabase/lib/metadata/calculation_test.cljc
@@ -1113,3 +1113,22 @@
"Orders → Created At"
"Orders → Quantity"]
(map :display-name (lib.metadata.result-metadata/returned-columns query)))))))
+
+(deftest ^:parallel visible-columns-include-sensitive-fields-test
+ (testing "visible-columns with :include-sensitive-fields? option"
+ (let [mp (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:fields [{:id (meta/id :venues :latitude)
+ :visibility-type :sensitive}
+ {:id (meta/id :venues :longitude)
+ :visibility-type :retired}]})
+ query (lib/query mp (lib.metadata/table mp (meta/id :venues)))]
+ (testing "sensitive column is NOT included by default"
+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query))]
+ (is (not (contains? visible-col-ids (meta/id :venues :latitude))))))
+ (testing "sensitive column IS included when :include-sensitive-fields? is true"
+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query -1 {:include-sensitive-fields? true}))]
+ (is (contains? visible-col-ids (meta/id :venues :latitude)))))
+ (testing "retired column is NOT included even with :include-sensitive-fields? true"
+ (let [visible-col-ids (into #{} (map :id) (lib/visible-columns query -1 {:include-sensitive-fields? true}))]
+ (is (not (contains? visible-col-ids (meta/id :venues :longitude)))))))))
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index e5a4a5853b3..4e215fabc27 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -1079,3 +1079,20 @@
(mt/with-current-user user-id
(mt/user-http-request user-id :post 403 "dataset"
(mt/mbql-query venues {:limit 1})))))))))))))
+
+(deftest query-metadata-sensitive-fields-test
+ (testing "POST /api/dataset/query_metadata"
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :price) {:visibility_type :sensitive}
+ (testing "sensitive fields are excluded by default"
+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/query_metadata"
+ (mt/mbql-query venues))]
+ (is (not (some #(= (:id %) (mt/id :venues :price))
+ (->> result :tables (mapcat :fields))))
+ "Sensitive field should NOT be included by default")))
+ (testing "sensitive fields are included when :settings :include-sensitive-fields is true"
+ (let [result (mt/user-http-request :crowberto :post 200 "dataset/query_metadata"
+ (assoc (mt/mbql-query venues)
+ :settings {:include_sensitive_fields true}))]
+ (is (some #(= (:id %) (mt/id :venues :price))
+ (->> result :tables (mapcat :fields)))
+ "Sensitive field SHOULD be included when :settings :include-sensitive-fields is true"))))))