Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

endpoint for exposting field values (#73003)

Aleksandr Lesnenkoec6e4baadb28
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index ce90ad13e67..d53ce7b3fdd 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2554,6 +2554,7 @@
:model/Collection
:model/Database
:model/Field
+ :model/FieldValues
:model/Table}}
xrays
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 1ccfbf07b45..c87ab718d51 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -604,6 +604,33 @@
reducible))
(.write writer "]"))
+(defn- visible-db-where
+ "Honeysql `:where` clause restricting a `:metabase_database :d` join to databases that
+ belong in a bulk export for the current user: not audit, not a router DB, and inside
+ the query-builder visibility filter."
+ []
+ [:and
+ [:= :d.is_audit false]
+ [:= :d.router_database_id nil]
+ [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])
+
+(defn- visible-table-where
+ "Honeysql `:where` clause restricting a `:metabase_table :t` join to active, non-hidden
+ tables inside the current user's query-builder visibility filter."
+ []
+ [:and
+ [:= :t.active true]
+ [:= :t.visibility_type nil]
+ [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])
+
+(defn- visible-field-where
+ "Honeysql `:where` clause restricting a `:metabase_field :f` join to active,
+ non-sensitive fields."
+ []
+ [:and
+ [:= :f.active true]
+ [:<> :f.visibility_type "sensitive"]])
+
(defn- write-databases-metadata!
"Streams the databases/tables/fields metadata as JSON to the given OutputStream.
@@ -612,17 +639,9 @@
section is written directly to the underlying writer as rows are pulled from a
reducible query, keeping memory usage bounded regardless of schema size."
[^java.io.OutputStream os]
- (let [db-filter [:and
- [:= :d.is_audit false]
- [:= :d.router_database_id nil]
- [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]]
- t-filter [:and
- [:= :t.active true]
- [:= :t.visibility_type nil]
- [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]]
- f-filter [:and
- [:= :f.active true]
- [:<> :f.visibility_type "sensitive"]]
+ (let [db-filter (visible-db-where)
+ t-filter (visible-table-where)
+ f-filter (visible-field-where)
writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
(.write writer "{\"databases\":")
(write-json-array! writer
@@ -696,6 +715,74 @@
(streaming-response {:content-type "application/json; charset=utf-8"} [os _]
(write-databases-metadata! os)))
+;;; --------------------------------------- GET /api/database/field-values ---------------------------------------
+
+(defn- format-field-values-entry
+ "Formats a FieldValues row for the /field-values response. Omits `human_readable_values`
+ when empty to keep the common case compact."
+ [{:keys [field_id values human_readable_values has_more_values]}]
+ (m/assoc-some {:field_id field_id
+ :values (or values [])
+ :has_more_values (boolean has_more_values)}
+ :human_readable_values (not-empty human_readable_values)))
+
+(defn- write-field-values!
+ "Streams the `field_values` JSON array to `os`. Exports only unconstrained (`:full`)
+ FieldValues — sandboxed, impersonation, and linked-filter variants are user-specific
+ and excluded from the bulk export. Visibility filter matches `GET /api/database/metadata`
+ so every streamed row has a corresponding field entry there."
+ [^java.io.OutputStream os]
+ (let [db-filter (visible-db-where)
+ t-filter (visible-table-where)
+ f-filter (visible-field-where)
+ fv-filter [:and
+ [:= :fv.type "full"]
+ [:= :fv.hash_key nil]]
+ writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
+ (.write writer "{\"field_values\":")
+ (write-json-array! writer
+ (t2/reducible-select [:model/FieldValues
+ :fv.field_id :fv.values :fv.human_readable_values :fv.has_more_values]
+ {:from [[:metabase_fieldvalues :fv]]
+ :join [[:metabase_field :f] [:= :fv.field_id :f.id]
+ [:metabase_table :t] [:= :f.table_id :t.id]
+ [:metabase_database :d] [:= :t.db_id :d.id]]
+ :where [:and db-filter t-filter f-filter fv-filter]})
+ format-field-values-entry)
+ (.write writer "}")
+ (.flush writer)))
+
+(mr/def ::field-values-info
+ [:map
+ [:field_id ::lib.schema.id/field]
+ [:values [:sequential [:sequential :any]]]
+ [:has_more_values :boolean]
+ [:human_readable_values {:optional true} [:sequential [:maybe :string]]]])
+
+(mr/def ::field-values-response
+ [:map
+ [:field_values [:sequential ::field-values-info]]])
+
+(api.macros/defendpoint :get "/field-values"
+ :- (server.streaming-response/streaming-response-schema ::field-values-response)
+ "Get sampled field values for every field in the instance, streamed as a single
+ `{\"field_values\": [...]}` document. Each entry carries `field_id`, `values`,
+ optional `human_readable_values`, and `has_more_values`.
+
+ Only unconstrained (`:full`) FieldValues are included — sandboxed, impersonation, and
+ linked-filter variants are user-specific and would bypass their own enforcement
+ mechanisms in a bulk export. Pair with `GET /api/database/metadata` to resolve
+ `field_id` to table and field names. Response is streamed for efficiency with large
+ schemas.
+
+ Admin-only: this endpoint exposes cached values computed over the unrestricted
+ dataset, so it would leak data past sandbox / impersonation rules if served to
+ regular users."
+ []
+ (api/check-superuser)
+ (streaming-response {:content-type "application/json; charset=utf-8"} [os _]
+ (write-field-values! os)))
+
;;; ----------------------------------------- POST /api/database/metadata -----------------------------------------
(def ^:private import-batch-size
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 643b0a59c7f..4b9b769e634 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -804,6 +804,66 @@
(is (= {:databases [] :tables [] :fields []}
(mt/user-http-request :rasta :get 202 "database/metadata")))))))
+(deftest databases-field-values-test
+ (testing "GET /api/database/field-values"
+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
+ :model/Field {f1-id :id} {:table_id t-id :name "state" :base_type :type/Text
+ :database_type "VARCHAR"}
+ :model/Field {f2-id :id} {:table_id t-id :name "rating" :base_type :type/Integer
+ :database_type "INTEGER"}
+ :model/FieldValues _ {:field_id f1-id :type :full
+ :values [["CA"] ["NY"] ["TX"]]
+ :has_more_values false}
+ :model/FieldValues _ {:field_id f2-id :type :full
+ :values [[1] [2] [3]]
+ :human_readable_values ["Low" "Mid" "High"]
+ :has_more_values true}]
+ (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")
+ by-field (into {} (map (juxt :field_id identity)) field_values)]
+ (is (=? {:field_id f1-id
+ :values [["CA"] ["NY"] ["TX"]]
+ :has_more_values false}
+ (by-field f1-id)))
+ (is (nil? (:human_readable_values (by-field f1-id)))
+ "human_readable_values is omitted when empty")
+ (is (=? {:field_id f2-id
+ :values [[1] [2] [3]]
+ :human_readable_values ["Low" "Mid" "High"]
+ :has_more_values true}
+ (by-field f2-id)))))))
+
+(deftest databases-field-values-non-admin-test
+ (testing "GET /api/database/field-values — non-admin requests are rejected"
+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
+ :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text
+ :database_type "VARCHAR"}
+ :model/FieldValues _ {:field_id f-id :type :full
+ :values [["CA"] ["NY"]]
+ :has_more_values false}]
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :get 403 "database/field-values"))))))
+
+(deftest databases-field-values-excludes-non-full-test
+ (testing "GET /api/database/field-values — only :full FieldValues are included"
+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
+ :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text
+ :database_type "VARCHAR"}
+ :model/FieldValues _ {:field_id f-id :type :full
+ :values [["CA"]]
+ :has_more_values false}
+ :model/FieldValues _ {:field_id f-id :type :sandbox
+ :hash_key "sandbox-hash"
+ :values [["NY"]]
+ :has_more_values false}]
+ (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")
+ for-field (filter #(= f-id (:field_id %)) field_values)]
+ (is (= 1 (count for-field))
+ "only the :full entry streams; :sandbox / other variants are excluded")
+ (is (= [["CA"]] (-> for-field first :values)))))))
+
(deftest databases-metadata-excludes-audit-db-test
(testing "GET /api/database/metadata — audit (internal) database, its tables, and its fields are excluded"
(mt/with-temp [:model/Database {db-id :id} {:name "audit-db" :engine :h2 :is_audit true}