The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
endpoint for exposting field values (#73003)
Aleksandr Lesnenkoec6e4baadb28
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ce90ad13e67..d53ce7b3fdd 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2554,6 +2554,7 @@:model/Collection:model/Database:model/Field+ :model/FieldValues:model/Table}}xraysdiff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 1ccfbf07b45..c87ab718d51 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -604,6 +604,33 @@reducible))(.write writer "]"))+(defn- visible-db-where+ "Honeysql `:where` clause restricting a `:metabase_database :d` join to databases that+ belong in a bulk export for the current user: not audit, not a router DB, and inside+ the query-builder visibility filter."+ []+ [:and+ [:= :d.is_audit false]+ [:= :d.router_database_id nil]+ [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])++(defn- visible-table-where+ "Honeysql `:where` clause restricting a `:metabase_table :t` join to active, non-hidden+ tables inside the current user's query-builder visibility filter."+ []+ [:and+ [:= :t.active true]+ [:= :t.visibility_type nil]+ [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])++(defn- visible-field-where+ "Honeysql `:where` clause restricting a `:metabase_field :f` join to active,+ non-sensitive fields."+ []+ [:and+ [:= :f.active true]+ [:<> :f.visibility_type "sensitive"]])+(defn- write-databases-metadata!"Streams the databases/tables/fields metadata as JSON to the given OutputStream.@@ -612,17 +639,9 @@section is written directly to the underlying writer as rows are pulled from areducible query, keeping memory usage bounded regardless of schema size."[^java.io.OutputStream os]- (let [db-filter [:and- [:= :d.is_audit false]- [:= :d.router_database_id nil]- [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]]- t-filter [:and- [:= :t.active true]- [:= :t.visibility_type nil]- [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]]- f-filter [:and- [:= :f.active true]- [:<> :f.visibility_type "sensitive"]]+ (let [db-filter (visible-db-where)+ t-filter (visible-table-where)+ f-filter (visible-field-where)writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))](.write writer "{\"databases\":")(write-json-array! writer@@ -696,6 +715,74 @@(streaming-response {:content-type "application/json; charset=utf-8"} [os _](write-databases-metadata! os)))+;;; --------------------------------------- GET /api/database/field-values ---------------------------------------++(defn- format-field-values-entry+ "Formats a FieldValues row for the /field-values response. Omits `human_readable_values`+ when empty to keep the common case compact."+ [{:keys [field_id values human_readable_values has_more_values]}]+ (m/assoc-some {:field_id field_id+ :values (or values [])+ :has_more_values (boolean has_more_values)}+ :human_readable_values (not-empty human_readable_values)))++(defn- write-field-values!+ "Streams the `field_values` JSON array to `os`. Exports only unconstrained (`:full`)+ FieldValues — sandboxed, impersonation, and linked-filter variants are user-specific+ and excluded from the bulk export. Visibility filter matches `GET /api/database/metadata`+ so every streamed row has a corresponding field entry there."+ [^java.io.OutputStream os]+ (let [db-filter (visible-db-where)+ t-filter (visible-table-where)+ f-filter (visible-field-where)+ fv-filter [:and+ [:= :fv.type "full"]+ [:= :fv.hash_key nil]]+ writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]+ (.write writer "{\"field_values\":")+ (write-json-array! writer+ (t2/reducible-select [:model/FieldValues+ :fv.field_id :fv.values :fv.human_readable_values :fv.has_more_values]+ {:from [[:metabase_fieldvalues :fv]]+ :join [[:metabase_field :f] [:= :fv.field_id :f.id]+ [:metabase_table :t] [:= :f.table_id :t.id]+ [:metabase_database :d] [:= :t.db_id :d.id]]+ :where [:and db-filter t-filter f-filter fv-filter]})+ format-field-values-entry)+ (.write writer "}")+ (.flush writer)))++(mr/def ::field-values-info+ [:map+ [:field_id ::lib.schema.id/field]+ [:values [:sequential [:sequential :any]]]+ [:has_more_values :boolean]+ [:human_readable_values {:optional true} [:sequential [:maybe :string]]]])++(mr/def ::field-values-response+ [:map+ [:field_values [:sequential ::field-values-info]]])++(api.macros/defendpoint :get "/field-values"+ :- (server.streaming-response/streaming-response-schema ::field-values-response)+ "Get sampled field values for every field in the instance, streamed as a single+ `{\"field_values\": [...]}` document. Each entry carries `field_id`, `values`,+ optional `human_readable_values`, and `has_more_values`.++ Only unconstrained (`:full`) FieldValues are included — sandboxed, impersonation, and+ linked-filter variants are user-specific and would bypass their own enforcement+ mechanisms in a bulk export. Pair with `GET /api/database/metadata` to resolve+ `field_id` to table and field names. Response is streamed for efficiency with large+ schemas.++ Admin-only: this endpoint exposes cached values computed over the unrestricted+ dataset, so it would leak data past sandbox / impersonation rules if served to+ regular users."+ []+ (api/check-superuser)+ (streaming-response {:content-type "application/json; charset=utf-8"} [os _]+ (write-field-values! os)))+;;; ----------------------------------------- POST /api/database/metadata -----------------------------------------(def ^:private import-batch-sizediff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 643b0a59c7f..4b9b769e634 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -804,6 +804,66 @@(is (= {:databases [] :tables [] :fields []}(mt/user-http-request :rasta :get 202 "database/metadata")))))))+(deftest databases-field-values-test+ (testing "GET /api/database/field-values"+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}+ :model/Field {f1-id :id} {:table_id t-id :name "state" :base_type :type/Text+ :database_type "VARCHAR"}+ :model/Field {f2-id :id} {:table_id t-id :name "rating" :base_type :type/Integer+ :database_type "INTEGER"}+ :model/FieldValues _ {:field_id f1-id :type :full+ :values [["CA"] ["NY"] ["TX"]]+ :has_more_values false}+ :model/FieldValues _ {:field_id f2-id :type :full+ :values [[1] [2] [3]]+ :human_readable_values ["Low" "Mid" "High"]+ :has_more_values true}]+ (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")+ by-field (into {} (map (juxt :field_id identity)) field_values)]+ (is (=? {:field_id f1-id+ :values [["CA"] ["NY"] ["TX"]]+ :has_more_values false}+ (by-field f1-id)))+ (is (nil? (:human_readable_values (by-field f1-id)))+ "human_readable_values is omitted when empty")+ (is (=? {:field_id f2-id+ :values [[1] [2] [3]]+ :human_readable_values ["Low" "Mid" "High"]+ :has_more_values true}+ (by-field f2-id)))))))++(deftest databases-field-values-non-admin-test+ (testing "GET /api/database/field-values — non-admin requests are rejected"+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}+ :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text+ :database_type "VARCHAR"}+ :model/FieldValues _ {:field_id f-id :type :full+ :values [["CA"] ["NY"]]+ :has_more_values false}]+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 "database/field-values"))))))++(deftest databases-field-values-excludes-non-full-test+ (testing "GET /api/database/field-values — only :full FieldValues are included"+ (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}+ :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text+ :database_type "VARCHAR"}+ :model/FieldValues _ {:field_id f-id :type :full+ :values [["CA"]]+ :has_more_values false}+ :model/FieldValues _ {:field_id f-id :type :sandbox+ :hash_key "sandbox-hash"+ :values [["NY"]]+ :has_more_values false}]+ (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")+ for-field (filter #(= f-id (:field_id %)) field_values)]+ (is (= 1 (count for-field))+ "only the :full entry streams; :sandbox / other variants are excluded")+ (is (= [["CA"]] (-> for-field first :values)))))))+(deftest databases-metadata-excludes-audit-db-test(testing "GET /api/database/metadata — audit (internal) database, its tables, and its fields are excluded"(mt/with-temp [:model/Database {db-id :id} {:name "audit-db" :engine :h2 :is_audit true}