Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

/api/database/metadata endpoint (#72683)

Alexander Polyankin6a318282289e
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f9bd1d9ba24..761135042da 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2536,6 +2536,7 @@
request
sample-data
secrets
+ server
settings
sync
upload
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 5d4f6fe0d4c..88252024ddc 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -28,6 +28,7 @@
[metabase.request.core :as request]
[metabase.sample-data.core :as sample-data]
[metabase.secrets.core :as secret]
+ [metabase.server.streaming-response :as server.streaming-response :refer [streaming-response]]
[metabase.settings.core :as setting]
[metabase.sync.core :as sync]
[metabase.sync.schedules :as sync.schedules]
@@ -37,6 +38,7 @@
[metabase.util.cron :as u.cron]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :as i18n :refer [deferred-tru trs tru]]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
@@ -544,6 +546,116 @@
:when query]
[query model])})))
+;;; ----------------------------------------- GET /api/database/metadata ------------------------------------------
+
+(defn- format-database-metadata
+ "Formats a Database record for the /metadata endpoint response."
+ [{:keys [id name engine]}]
+ {:id id :name name :engine engine})
+
+(defn- format-table-metadata
+ "Formats a Table record for the /metadata endpoint response, omitting nil optional fields."
+ [{:keys [id db_id name schema description]}]
+ (m/assoc-some {:id id :db_id db_id :name name}
+ :schema schema
+ :description description))
+
+(defn- perm-user-info []
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*})
+
+(defn- format-field-metadata
+ "Formats a Field record for the /metadata endpoint response. Includes effective_type only when it differs from base_type."
+ [{:keys [id table_id parent_id fk_target_field_id name description base_type database_type effective_type semantic_type coercion_strategy]}]
+ (m/assoc-some {:id id :table_id table_id :name name}
+ :parent_id parent_id
+ :fk_target_field_id fk_target_field_id
+ :description description
+ :base_type base_type
+ :database_type database_type
+ :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
+ :semantic_type semantic_type
+ :coercion_strategy coercion_strategy))
+
+(defn- perm-mapping []
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})
+
+(defn- write-json-array!
+ "Streams a reducible collection as a JSON array to a Writer, applying `format-fn` to each row."
+ [^java.io.Writer writer reducible format-fn]
+ (.write writer "[")
+ (let [first? (volatile! true)]
+ (reduce (fn [_ row]
+ (if @first?
+ (vreset! first? false)
+ (.write writer ","))
+ (json/encode-to (format-fn row) writer {}))
+ nil
+ reducible))
+ (.write writer "]"))
+
+(defn- write-databases-metadata!
+ "Streams the databases/tables/fields metadata as JSON to the given OutputStream."
+ [^java.io.OutputStream os]
+ (let [db-filter [:and [:= :is_audit false] [:= :router_database_id nil]
+ [:in :id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]]
+ t-filter [:and [:= :active true] [:= :visibility_type nil]
+ [:in :id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]]
+ f-filter [:and [:= :active true] [:<> :visibility_type "sensitive"]
+ [:in :table_id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]]
+ writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
+ (.write writer "{\"databases\":")
+ (write-json-array! writer
+ (t2/reducible-select [:model/Database :id :name :engine] {:where db-filter})
+ format-database-metadata)
+ (.write writer ",\"tables\":")
+ (write-json-array! writer
+ (t2/reducible-select [:model/Table :id :db_id :name :schema :description] {:where t-filter})
+ format-table-metadata)
+ (.write writer ",\"fields\":")
+ (write-json-array! writer
+ (t2/reducible-select [:model/Field :id :table_id :parent_id :fk_target_field_id :name :description
+ :base_type :database_type :effective_type :semantic_type :coercion_strategy]
+ {:where f-filter})
+ format-field-metadata)
+ (.write writer "}")
+ (.flush writer)))
+
+(mr/def ::databases-metadata-response
+ [:map
+ [:databases [:sequential [:map
+ [:id ::lib.schema.id/database]
+ [:name :string]
+ [:engine :string]]]]
+ [:tables [:sequential [:map
+ [:id ::lib.schema.id/table]
+ [:db_id ::lib.schema.id/database]
+ [:name :string]
+ [:schema {:optional true} :string]
+ [:description {:optional true} :string]]]]
+ [:fields [:sequential [:map
+ [:id ::lib.schema.id/field]
+ [:table_id ::lib.schema.id/table]
+ [:name :string]
+ [:parent_id {:optional true} ::lib.schema.id/field]
+ [:fk_target_field_id {:optional true} ::lib.schema.id/field]
+ [:description {:optional true} :string]
+ [:base_type :string]
+ [:database_type {:optional true} :string]
+ [:effective_type {:optional true} :string]
+ [:semantic_type {:optional true} :string]
+ [:coercion_strategy {:optional true} :string]]]]])
+
+(api.macros/defendpoint :get "/metadata"
+ :- (server.streaming-response/streaming-response-schema ::databases-metadata-response)
+ "Get metadata (databases, tables, and fields) for all databases visible to the current user.
+ Returns a flat structure with three arrays: databases, tables, and fields.
+ Response is streamed for efficiency with large schemas."
+ []
+ (streaming-response {:content-type "application/json; charset=utf-8"} [os _]
+ (write-databases-metadata! os)))
+
;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------
;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbers
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index c302b91a3f7..105d291ffe4 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -749,6 +749,61 @@
(is (true? @connections-stay-open?))
(tx/destroy-db! driver/*driver* empty-dbdef)))))
+(deftest databases-metadata-test
+ (testing "GET /api/database/metadata"
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"
+ :description "A test table"}
+ :model/Field {f1-id :id} {:table_id t-id :name "id" :base_type :type/Integer
+ :database_type "BIGINT"
+ :semantic_type :type/PK}
+ :model/Field {f2-id :id} {:table_id t-id :name "created_at" :base_type :type/Text
+ :database_type "TIMESTAMP"
+ :effective_type :type/DateTime
+ :semantic_type :type/Name
+ :coercion_strategy :Coercion/ISO8601->DateTime
+ :description "The creation time"}
+ :model/Field {f3-id :id} {:table_id t-id :name "parent_id" :base_type :type/Integer
+ :database_type "BIGINT"
+ :semantic_type :type/FK
+ :fk_target_field_id f1-id}]
+ (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202 "database/metadata")]
+ (is (=? {:id db-id :name "test-db" :engine "h2"}
+ (m/find-first (comp #{db-id} :id) databases)))
+ (is (=? {:id t-id :db_id db-id :name "my_table" :schema "PUBLIC" :description "A test table"}
+ (m/find-first (comp #{t-id} :id) tables)))
+ (is (=? {:id f1-id :table_id t-id :name "id" :base_type "type/Integer" :database_type "BIGINT"
+ :semantic_type "type/PK"}
+ (m/find-first (comp #{f1-id} :id) fields)))
+ (is (=? {:id f2-id
+ :table_id t-id
+ :name "created_at"
+ :base_type "type/Text"
+ :database_type "TIMESTAMP"
+ :effective_type "type/DateTime"
+ :semantic_type "type/Name"
+ :coercion_strategy "Coercion/ISO8601->DateTime"
+ :description "The creation time"}
+ (m/find-first (comp #{f2-id} :id) fields)))
+ (is (=? {:id f3-id
+ :table_id t-id
+ :name "parent_id"
+ :base_type "type/Integer"
+ :database_type "BIGINT"
+ :semantic_type "type/FK"
+ :fk_target_field_id f1-id}
+ (m/find-first (comp #{f3-id} :id) fields)))))))
+
+(deftest databases-metadata-no-perms-test
+ (testing "GET /api/database/metadata — user without data perms sees nothing"
+ (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"}
+ :model/Field _ {:table_id t-id :name "id" :base_type :type/Integer
+ :database_type "BIGINT"}]
+ (mt/with-no-data-perms-for-all-users!
+ (is (= {:databases [] :tables [] :fields []}
+ (mt/user-http-request :rasta :get 202 "database/metadata")))))))
+
(deftest ^:parallel fetch-database-metadata-test
(testing "GET /api/database/:id/metadata"
(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)