Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

External metadata (#73612)

Alexander Polyankin72975d87c3db
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 4f4bb54cbe2..69f85e4a6f4 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2588,7 +2588,6 @@
request
sample-data
secrets
- server
settings
sync
upload
@@ -2600,7 +2599,6 @@
:model/Collection
:model/Database
:model/Field
- :model/FieldValues
:model/Table}}
xrays
@@ -3303,6 +3301,7 @@
lib
logger
models
+ permissions
plugins
premium-features
search
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index a51cb93d7e3..32b3e195a77 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -3,6 +3,8 @@
[clojure.java.io :as io]
[clojure.string :as str]
[java-time.api :as t]
+ [metabase-enterprise.serialization.metadata :as metadata]
+ [metabase-enterprise.serialization.schema :as schema]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase-enterprise.serialization.v2.load :as v2.load]
@@ -20,6 +22,7 @@
[metabase.util :as u]
[metabase.util.compress :as u.compress]
[metabase.util.date-2 :as u.date]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -306,6 +309,89 @@
(finally
(io/delete-file (:tempfile file)))))
+;;; ----------------------------------- GET /api/ee/serialization/metadata/export -----------------------------------
+
+(defn- write-json-array!
+ "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded
+ directly with no transformation — apply per-row formatting via an `eduction` (or other
+ transducer pipeline) before passing in.
+
+ `run!` is required here because it dispatches through `reduce`, which consumes the
+ `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.
+ `doseq` cannot be used: it walks a seq, and producing a seq from the reducible
+ would realize every row into memory — defeating the point of streaming."
+ [^java.io.Writer writer reducible]
+ (.write writer "[")
+ (let [first? (volatile! true)]
+ (run! (fn [row]
+ (if @first?
+ (vreset! first? false)
+ (.write writer ","))
+ (json/encode-to row writer {}))
+ reducible))
+ (.write writer "]"))
+
+(defn- write-json-object!
+ "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible
+ of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)
+ of already-formatted values to encode.
+
+ `run!` is used over the entries — `doseq` would walk a seq and realize the underlying
+ reducible, defeating streaming."
+ [^java.io.Writer writer entries]
+ (.write writer "{")
+ (let [first? (volatile! true)]
+ (run! (fn [[entry-name objects]]
+ (if @first? (vreset! first? false) (.write writer ","))
+ (.write writer (str "\"" entry-name "\":"))
+ (write-json-array! writer objects))
+ entries))
+ (.write writer "}"))
+
+(defn- write-databases-metadata!
+ "Streams the databases/tables/fields metadata to the given OutputStream. Sections are
+ included only when their `with-...?` flag is true. All references — database, table,
+ fk_target_field — are emitted in serdes-portable form (names rather than numeric IDs)
+ so the response can be ingested by another Metabase instance with different surrogate
+ keys.
+
+ Warehouses with large schemas can produce gigabytes of metadata, so rows are pulled
+ from reducible queries and streamed directly to the writer — memory stays bounded
+ regardless of schema size."
+ [^java.io.OutputStream os {:keys [with-databases? with-tables? with-fields?]}]
+ (let [writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
+ (write-json-object!
+ writer
+ (cond-> []
+ with-databases? (conj ["databases" (metadata/reducible-databases)])
+ with-tables? (conj ["tables" (metadata/reducible-tables)])
+ with-fields? (conj ["fields" (metadata/reducible-fields)])))
+ (.flush writer)))
+
+(api.macros/defendpoint :get "/metadata/export"
+ :- (sr/streaming-response-schema ::schema/metadata-export-response)
+ "Get warehouse metadata (databases, tables, and fields) for all databases visible to the
+ current user, with all references emitted in serdes-portable form (database names, table
+ `[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).
+
+ Sections must be opted into with the `with-databases`, `with-tables`, and `with-fields`
+ query parameters — they all default to `false`. The response is streamed for efficiency
+ with large schemas.
+
+ Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or
+ `Query builder and native`) permissions on each database and table."
+ [_route-params
+ {:keys [with-databases with-tables with-fields]}
+ :- [:map
+ [:with-databases {:default false} [:maybe :boolean]]
+ [:with-tables {:default false} [:maybe :boolean]]
+ [:with-fields {:default false} [:maybe :boolean]]]]
+ (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]
+ (write-databases-metadata! os
+ {:with-databases? with-databases
+ :with-tables? with-tables
+ :with-fields? with-fields})))
+
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/serialization` routes."
(api.macros/ns-handler *ns* +auth))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj
new file mode 100644
index 00000000000..22e459d4e16
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj
@@ -0,0 +1,187 @@
+(ns metabase-enterprise.serialization.metadata
+ "Reducible queries and row formatters that produce serdes-portable warehouse metadata
+ (databases, tables, and fields) for the `/api/ee/serialization/metadata/export` endpoint.
+ All references — database, table, fk_target_field — are emitted as portable references
+ (names rather than numeric IDs) so the response can be ingested by another Metabase
+ instance with different surrogate keys."
+ (:require
+ [medley.core :as m]
+ [metabase.api.common :as api]
+ [metabase.permissions.core :as perms]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(defn- perm-user-info
+ "Current-user info passed to `perms/visible-*-filter-select` to scope the visibility filters."
+ []
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*})
+
+(defn- perm-mapping
+ "Permission mapping required to include a database/table in the export."
+ []
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})
+
+(defn- visible-db-where
+ "Honeysql predicate selecting non-audit, non-router databases visible to the current user.
+ `d` is the table alias of the joined `metabase_database`."
+ [d]
+ [:and
+ [:= (u/qualified-key d :is_audit) false]
+ [:= (u/qualified-key d :router_database_id) nil]
+ [:in (u/qualified-key d :id) (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])
+
+(defn- visible-table-where
+ "Honeysql predicate selecting active, non-hidden tables visible to the current user.
+ `t` is the table alias of the joined `metabase_table`."
+ [t]
+ [:and
+ [:= (u/qualified-key t :active) true]
+ [:= (u/qualified-key t :visibility_type) nil]
+ [:in (u/qualified-key t :id) (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])
+
+(defn- visible-field-where
+ "Honeysql predicate selecting active, non-sensitive fields. `f` is the table alias of the
+ joined `metabase_field`."
+ [f]
+ [:and
+ [:= (u/qualified-key f :active) true]
+ [:<> (u/qualified-key f :visibility_type) "sensitive"]])
+
+(defn- reducible-databases-query
+ "Raw reducible-query streaming visible-database rows for [[reducible-databases]]."
+ []
+ (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]
+ :from [[:metabase_database :db]]
+ :where (visible-db-where :db)}))
+
+(defn- reducible-tables-query
+ "Raw reducible-query streaming visible-table rows for [[reducible-tables]]."
+ []
+ (t2/reducible-query {:select [[:db.name :db_name]
+ [:table.schema :schema]
+ [:table.name :table_name]
+ [:table.description :description]]
+ :from [[:metabase_table :table]]
+ :join [[:metabase_database :db] [:= :table.db_id :db.id]]
+ :where [:and (visible-db-where :db) (visible-table-where :table)]}))
+
+(defn- reducible-fields-query
+ "Raw reducible-query streaming visible-field rows for [[reducible-fields]]. Visibility
+ filters on the FK target chain are folded into the LEFT JOIN ON clauses (rather than the
+ WHERE) so that an inaccessible target fails the join and the `fk_*` columns come back as
+ NULL — [[format-field-row]] then drops the `:fk_target_field_id` for that row."
+ []
+ (t2/reducible-query
+ {:select [[:db.name :db_name]
+ [:table.schema :table_schema]
+ [:table.name :table_name]
+ [:field.name :field_name]
+ [:field.description :description]
+ [:field.base_type :base_type]
+ [:field.database_type :database_type]
+ [:field.effective_type :effective_type]
+ [:field.semantic_type :semantic_type]
+ [:field.coercion_strategy :coercion_strategy]
+ [:field.nfc_path :nfc_path]
+ [:fk_db.name :fk_db_name]
+ [:fk_table.schema :fk_table_schema]
+ [:fk_table.name :fk_table_name]
+ [:fk_field.name :fk_field_name]
+ [:fk_field.nfc_path :fk_field_nfc_path]]
+ :from [[:metabase_field :field]]
+ :join [[:metabase_table :table] [:= :field.table_id :table.id]
+ [:metabase_database :db] [:= :table.db_id :db.id]]
+ :left-join [[:metabase_field :fk_field] [:and [:= :field.fk_target_field_id :fk_field.id] (visible-field-where :fk_field)]
+ [:metabase_table :fk_table] [:and [:= :fk_field.table_id :fk_table.id] (visible-table-where :fk_table)]
+ [:metabase_database :fk_db] [:and [:= :fk_table.db_id :fk_db.id] (visible-db-where :fk_db)]]
+ :where [:and (visible-db-where :db) (visible-table-where :table) (visible-field-where :field)]}))
+
+(defn- decode-nfc-path
+ "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for
+ null/empty, the original value when it's already a sequential collection, or the decoded
+ vector when it's a JSON string."
+ [nfc-path]
+ (cond
+ (nil? nfc-path) nil
+ (sequential? nfc-path) (seq nfc-path)
+ (string? nfc-path) (seq (json/decode nfc-path))))
+
+(defn- external-database-id
+ "Portable id for a database — its name."
+ [db-name]
+ db-name)
+
+(defn- external-table-id
+ "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."
+ [db-name schema table-name]
+ [db-name schema table-name])
+
+(defn- external-field-id
+ "Portable id for a field. When `nfc-path` is set (JSON-nested column), the path replaces
+ the field's display name; the field's own name is the leaf of `nfc-path` joined with
+ arrows, so the path is the canonical structural representation."
+ [db-name schema table-name field-name nfc-path]
+ (if (seq nfc-path)
+ (into [db-name schema table-name] nfc-path)
+ [db-name schema table-name field-name]))
+
+(defn format-database-row
+ "JSON shape for one database row, identifying the database by its portable id."
+ [{:keys [name engine]}]
+ {:id (external-database-id name) :name name :engine engine})
+
+(defn format-table-row
+ "JSON shape for one table row, with portable references for `:id` and `:db_id`. Optional
+ fields are omitted when nil."
+ [{:keys [db_name schema table_name description]}]
+ (m/assoc-some {:id (external-table-id db_name schema table_name)
+ :db_id (external-database-id db_name)
+ :name table_name}
+ :schema schema
+ :description description))
+
+(defn format-field-row
+ "JSON shape for one field row, with portable references for `:id`, `:table_id`, `:parent_id`,
+ and `:fk_target_field_id`. `:parent_id` is derived from `nfc_path` (the portable id with the
+ last path element dropped) and is only emitted when `nfc_path` has at least two elements.
+ Optional fields are omitted when nil."
+ [{:keys [db_name table_schema table_name field_name description base_type database_type
+ effective_type semantic_type coercion_strategy nfc_path
+ fk_db_name fk_table_schema fk_table_name fk_field_name fk_field_nfc_path]}]
+ (let [nfc-path (decode-nfc-path nfc_path)
+ parent-id (when-some [parent-nfc (seq (butlast nfc-path))]
+ (external-field-id db_name table_schema table_name nil parent-nfc))
+ fk-field-nfc (decode-nfc-path fk_field_nfc_path)
+ fk-target-id (when (and fk_db_name fk_table_name fk_field_name)
+ (external-field-id fk_db_name fk_table_schema fk_table_name
+ fk_field_name fk-field-nfc))]
+ (m/assoc-some {:id (external-field-id db_name table_schema table_name field_name nfc-path)
+ :table_id (external-table-id db_name table_schema table_name)
+ :name field_name}
+ :description description
+ :base_type base_type
+ :database_type database_type
+ :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
+ :semantic_type semantic_type
+ :coercion_strategy coercion_strategy
+ :nfc_path nfc-path
+ :parent_id parent-id
+ :fk_target_field_id fk-target-id)))
+
+(defn reducible-databases
+ "Eduction streaming visible databases as already-formatted JSON-shaped rows."
+ []
+ (eduction (map format-database-row) (reducible-databases-query)))
+
+(defn reducible-tables
+ "Eduction streaming visible tables as already-formatted JSON-shaped rows."
+ []
+ (eduction (map format-table-row) (reducible-tables-query)))
+
+(defn reducible-fields
+ "Eduction streaming visible fields as already-formatted JSON-shaped rows."
+ []
+ (eduction (map format-field-row) (reducible-fields-query)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
new file mode 100644
index 00000000000..740ec109256
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
@@ -0,0 +1,49 @@
+(ns metabase-enterprise.serialization.schema
+ "Malli schemas for the serdes-portable shapes produced by
+ [[metabase-enterprise.serialization.metadata]]."
+ (:require
+ [metabase.util.malli.registry :as mr]))
+
+(mr/def ::external-database-id
+ :string)
+
+(mr/def ::external-table-id
+ [:tuple :string [:maybe :string] :string])
+
+(mr/def ::external-field-id
+ [:cat :string [:maybe :string] :string [:+ :string]])
+
+(mr/def ::database-info
+ [:map
+ [:id ::external-database-id]
+ [:name :string]
+ [:engine :string]])
+
+(mr/def ::table-info
+ [:map
+ [:id ::external-table-id]
+ [:db_id ::external-database-id]
+ [:name :string]
+ [:schema {:optional true} :string]
+ [:description {:optional true} :string]])
+
+(mr/def ::field-info
+ [:map
+ [:id ::external-field-id]
+ [:table_id ::external-table-id]
+ [:name :string]
+ [:parent_id {:optional true} ::external-field-id]
+ [:fk_target_field_id {:optional true} ::external-field-id]
+ [:description {:optional true} :string]
+ [:base_type :string]
+ [:database_type {:optional true} :string]
+ [:effective_type {:optional true} :string]
+ [:semantic_type {:optional true} :string]
+ [:coercion_strategy {:optional true} :string]
+ [:nfc_path {:optional true} [:sequential :string]]])
+
+(mr/def ::metadata-export-response
+ [:map
+ [:databases {:optional true} [:sequential ::database-info]]
+ [:tables {:optional true} [:sequential ::table-info]]
+ [:fields {:optional true} [:sequential ::field-info]]])
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 884da4a7a22..56a212ef83a 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -4,10 +4,12 @@
[clojure.string :as str]
[clojure.test :refer :all]
[clojure.walk :as walk]
+ [medley.core :as m]
[metabase-enterprise.serialization.api :as api.serialization]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase.analytics.snowplow-test :as snowplow-test]
[metabase.models.serialization :as serdes]
+ [metabase.permissions.core :as perms]
[metabase.search.core :as search]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
@@ -441,6 +443,289 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file (byte-array 0)})))))))
+(deftest metadata-export-basic-test
+ (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
+ :description "A test table"}
+ :model/Field {f-name :name} {:table_id t-id
+ :base_type :type/Integer
+ :database_type "BIGINT"
+ :semantic_type :type/PK}]
+ (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-databases true
+ :with-tables true
+ :with-fields true)
+ test-db (m/find-first (comp #{db-name} :id) databases)
+ test-table (m/find-first (comp #{[db-name "PUBLIC" t-name]} :id) tables)
+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]
+ (is (=? {:id db-name :name db-name :engine "h2"}
+ test-db))
+ (is (=? {:id [db-name "PUBLIC" t-name]
+ :db_id db-name
+ :name t-name
+ :schema "PUBLIC"
+ :description "A test table"}
+ test-table))
+ (is (=? {:id [db-name "PUBLIC" t-name f-name]
+ :table_id [db-name "PUBLIC" t-name]
+ :name f-name
+ :base_type "type/Integer"
+ :database_type "BIGINT"
+ :semantic_type "type/PK"}
+ test-field)))))))
+
+(deftest metadata-export-optional-properties-test
+ (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {f-name :name} {:table_id t-id
+ :base_type :type/Text
+ :database_type "VARCHAR"
+ :effective_type :type/DateTime
+ :coercion_strategy :Coercion/ISO8601->DateTime
+ :description "When this happened"}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]
+ (is (=? {:id [db-name "PUBLIC" t-name f-name]
+ :name f-name
+ :base_type "type/Text"
+ :effective_type "type/DateTime"
+ :coercion_strategy "Coercion/ISO8601->DateTime"
+ :description "When this happened"}
+ test-field)))))))
+
+(deftest metadata-export-parent-field-test
+ (testing "GET /api/ee/serialization/metadata/export — parent_id is nfc_path with the last element dropped, omitted for root fields"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {root-name :name} {:table_id t-id
+ :base_type :type/Text}
+ :model/Field _ {:table_id t-id
+ :base_type :type/Text
+ :nfc_path ["data" "city"]}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name root-name]} :id) fields)
+ test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name "data" "city"]} :id) fields)]
+ (is (=? {:id [db-name "PUBLIC" t-name root-name]} test-field-1))
+ (is (not (contains? test-field-1 :parent_id)))
+ (is (=? {:id [db-name "PUBLIC" t-name "data" "city"]
+ :parent_id [db-name "PUBLIC" t-name "data"]}
+ test-field-2)))))))
+
+(deftest metadata-export-fk-test
+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted in portable form"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {pk-id :id pk-name :name} {:table_id t-id
+ :base_type :type/Integer
+ :semantic_type :type/PK}
+ :model/Field {fk-name :name} {:table_id t-id
+ :base_type :type/Integer
+ :semantic_type :type/FK
+ :fk_target_field_id pk-id}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name pk-name]} :id) fields)
+ test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]
+ (is (=? {:id [db-name "PUBLIC" t-name pk-name] :semantic_type "type/PK"}
+ test-field-1))
+ (is (=? {:id [db-name "PUBLIC" t-name fk-name]
+ :table_id [db-name "PUBLIC" t-name]
+ :semantic_type "type/FK"
+ :fk_target_field_id (:id test-field-1)}
+ test-field-2)))))))
+
+(deftest metadata-export-hidden-table-test
+ (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {hidden-id :id hidden-t-name :name} {:db_id db-id :schema "PUBLIC"
+ :visibility_type :hidden}
+ :model/Field {visible-f-name :name} {:table_id visible-id
+ :base_type :type/Integer}
+ :model/Field {hidden-f-name :name} {:table_id hidden-id
+ :base_type :type/Integer}]
+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-tables true :with-fields true)]
+ (is (some (comp #{visible-t-name} :name) tables))
+ (is (not-any? (comp #{hidden-t-name} :name) tables))
+ (is (some (comp #{visible-f-name} :name) fields))
+ (is (not-any? (comp #{hidden-f-name} :name) fields)))))))
+
+(deftest metadata-export-hidden-table-fk-test
+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is hidden"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"
+ :visibility_type :hidden}
+ :model/Field {target-id :id} {:table_id hidden-id
+ :base_type :type/Integer
+ :semantic_type :type/PK}
+ :model/Field {fk-name :name} {:table_id visible-id
+ :base_type :type/Integer
+ :semantic_type :type/FK
+ :fk_target_field_id target-id}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field (m/find-first (comp #{[db-name "PUBLIC" v-name fk-name]} :id) fields)]
+ (is (some? test-field))
+ (is (not (contains? test-field :fk_target_field_id))))))))
+
+(deftest metadata-export-inactive-table-test
+ (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
+ :active false}
+ :model/Field {active-f-name :name} {:table_id t-id
+ :base_type :type/Integer}
+ :model/Field {inactive-f-name :name} {:table_id t-id
+ :base_type :type/Integer
+ :active false}]
+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-tables true :with-fields true)]
+ (is (not-any? (comp #{t-name} :name) tables))
+ (is (not-any? (comp #{active-f-name} :name) fields))
+ (is (not-any? (comp #{inactive-f-name} :name) fields)))))))
+
+(deftest metadata-export-sensitive-field-test
+ (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {email-name :name} {:table_id t-id :base_type :type/Text}
+ :model/Field {secret-name :name} {:table_id t-id :base_type :type/Text
+ :visibility_type :sensitive}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)]
+ (is (some (comp #{email-name} :name) fields))
+ (is (not-any? (comp #{secret-name} :name) fields)))))))
+
+(deftest metadata-export-sensitive-fk-test
+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target field is sensitive"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {target-id :id} {:table_id t-id
+ :base_type :type/Text
+ :visibility_type :sensitive}
+ :model/Field {fk-name :name} {:table_id t-id
+ :base_type :type/Text
+ :semantic_type :type/FK
+ :fk_target_field_id target-id}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]
+ (is (some? test-field))
+ (is (not (contains? test-field :fk_target_field_id))))))))
+
+(deftest metadata-export-inactive-field-test
+ (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {active-f-name :name} {:table_id t-id
+ :base_type :type/Integer}
+ :model/Field {inactive-f-name :name} {:table_id t-id
+ :base_type :type/Integer
+ :active false}]
+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-tables true :with-fields true)]
+ (is (some (comp #{t-name} :name) tables))
+ (is (some (comp #{active-f-name} :name) fields))
+ (is (not-any? (comp #{inactive-f-name} :name) fields)))))))
+
+(deftest metadata-export-db-routing-test
+ (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database _ {:engine :h2}
+ :model/Database {primary-id :id primary-name :name} {:engine :h2}
+ :model/Database {mirror-name :name} {:engine :h2
+ :router_database_id primary-id}]
+ (let [{:keys [databases]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
+ :with-databases true)]
+ (is (some (comp #{primary-name} :name) databases))
+ (is (not-any? (comp #{mirror-name} :name) databases)))))))
+
+(deftest metadata-export-table-permission-test
+ (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {restricted-id :id restricted-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {ok-f-name :name} {:table_id accessible-id
+ :base_type :type/Integer}
+ :model/Field {no-f-name :name} {:table_id restricted-id
+ :base_type :type/Integer}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
+ (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
+ (let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202
+ "ee/serialization/metadata/export"
+ :with-tables true :with-fields true)]
+ (is (some (comp #{accessible-name} :name) tables))
+ (is (not-any? (comp #{restricted-name} :name) tables))
+ (is (some (comp #{ok-f-name} :name) fields))
+ (is (not-any? (comp #{no-f-name} :name) fields)))))))
+
+(deftest metadata-export-table-permission-fk-test
+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is not accessible"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
+ :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {target-id :id} {:table_id restricted-id
+ :base_type :type/Integer
+ :semantic_type :type/PK}
+ :model/Field {fk-name :name} {:table_id accessible-id
+ :base_type :type/Integer
+ :semantic_type :type/FK
+ :fk_target_field_id target-id}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
+ (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
+ (let [{:keys [fields]} (mt/user-http-request :rasta :get 202
+ "ee/serialization/metadata/export"
+ :with-fields true)
+ test-field (m/find-first (comp #{[db-name "PUBLIC" accessible-name fk-name]} :id) fields)]
+ (is (some? test-field))
+ (is (not (contains? test-field :fk_target_field_id))))))))
+
+(deftest metadata-export-token-feature-test
+ (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"
+ (mt/with-premium-features #{}
+ (mt/assert-has-premium-feature-error
+ "Serialization"
+ (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))
+
(deftest serialization-cleanup-test
(testing "No temp files are left behind after export/import operations"
(let [known-files (set (.list (io/file api.serialization/parent-dir)))]
diff --git a/src/metabase/util.cljc b/src/metabase/util.cljc
index a1a144a6ab3..d95da88151f 100644
--- a/src/metabase/util.cljc
+++ b/src/metabase/util.cljc
@@ -57,6 +57,7 @@
format-nanoseconds
format-seconds
format-plural
+ qualified-key
qualified-name])
#?(:clj (p/import-vars [u.jvm
diff --git a/src/metabase/util/format.cljc b/src/metabase/util/format.cljc
index 60be7ea69bd..aa675ed8730 100644
--- a/src/metabase/util/format.cljc
+++ b/src/metabase/util/format.cljc
@@ -103,6 +103,11 @@
plural
(str singular \s)))))
+(defn qualified-key
+ "(u/qualified-key :t :id) -> :t.id"
+ [a b]
+ (keyword (str (name a) "." (name b))))
+
(defn qualified-name
"Return `k` as a string, qualified by its namespace, if any (unlike `name`). Handles `nil` values gracefully as well
(also unlike `name`).
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index b30943d807d..d7d98941f89 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -28,7 +28,6 @@
[metabase.request.core :as request]
[metabase.sample-data.core :as sample-data]
[metabase.secrets.core :as secret]
- [metabase.server.streaming-response :as server.streaming-response :refer [streaming-response]]
[metabase.settings.core :as setting]
[metabase.sync.core :as sync]
[metabase.sync.schedules :as sync.schedules]
@@ -38,7 +37,6 @@
[metabase.util.cron :as u.cron]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :as i18n :refer [deferred-tru trs tru]]
- [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
@@ -546,243 +544,6 @@
:when query]
[query model])})))
-;;; ----------------------------------------- GET /api/database/metadata ------------------------------------------
-
-(defn- format-database-metadata
- "Formats a Database record for the /metadata endpoint response."
- [{:keys [id name engine]}]
- {:id id :name name :engine engine})
-
-(defn- format-table-metadata
- "Formats a Table record for the /metadata endpoint response, omitting nil optional fields."
- [{:keys [id db_id name schema description]}]
- (m/assoc-some {:id id :db_id db_id :name name}
- :schema schema
- :description description))
-
-(defn- format-field-metadata
- "Formats a Field record for the /metadata endpoint response. Includes effective_type only when it differs from base_type."
- [{:keys [id table_id parent_id fk_target_field_id name description base_type database_type effective_type semantic_type coercion_strategy]}]
- (m/assoc-some {:id id :table_id table_id :name name}
- :parent_id parent_id
- :fk_target_field_id fk_target_field_id
- :description description
- :base_type base_type
- :database_type database_type
- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
- :semantic_type semantic_type
- :coercion_strategy coercion_strategy))
-
-(defn- perm-user-info
- "User information used to check permissions."
- []
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*})
-
-(defn- perm-mapping
- "Permission mapping used to filter databases and tables to those visible to the current user.
- Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or `Query builder and native`)."
- []
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder})
-
-(defn- write-json-array!
- "Streams a reducible collection as a JSON array to a Writer, applying `format-fn` to each row.
-
- `run!` is required here because it dispatches through `reduce`, which consumes the
- `IReduceInit` returned by `t2/reducible-select` row-by-row without materializing.
- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible
- would realize every row into memory — defeating the point of streaming."
- [^java.io.Writer writer reducible format-fn]
- (.write writer "[")
- (let [first? (volatile! true)]
- (run! (fn [row]
- (if @first?
- (vreset! first? false)
- (.write writer ","))
- (json/encode-to (format-fn row) writer {}))
- reducible))
- (.write writer "]"))
-
-(defn- visible-db-where
- "Honeysql `:where` clause restricting a `:metabase_database :d` join to databases that
- belong in a bulk export for the current user: not audit, not a router DB, and inside
- the query-builder visibility filter."
- []
- [:and
- [:= :d.is_audit false]
- [:= :d.router_database_id nil]
- [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])
-
-(defn- visible-table-where
- "Honeysql `:where` clause restricting a `:metabase_table :t` join to active, non-hidden
- tables inside the current user's query-builder visibility filter."
- []
- [:and
- [:= :t.active true]
- [:= :t.visibility_type nil]
- [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])
-
-(defn- visible-field-where
- "Honeysql `:where` clause restricting a `:metabase_field :f` join to active,
- non-sensitive fields."
- []
- [:and
- [:= :f.active true]
- [:<> :f.visibility_type "sensitive"]])
-
-(defn- write-databases-metadata!
- "Streams the databases/tables/fields metadata as JSON to the given OutputStream.
-
- Warehouses with large schemas can produce gigabytes of metadata, so streaming is
- required — materializing the full response in memory would OOM the server. Each
- section is written directly to the underlying writer as rows are pulled from a
- reducible query, keeping memory usage bounded regardless of schema size."
- [^java.io.OutputStream os]
- (let [db-filter (visible-db-where)
- t-filter (visible-table-where)
- f-filter (visible-field-where)
- writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
- (.write writer "{\"databases\":")
- (write-json-array! writer
- (t2/reducible-select [:model/Database :d.id :d.name :d.engine]
- {:from [[:metabase_database :d]]
- :where db-filter})
- format-database-metadata)
- (.write writer ",\"tables\":")
- (write-json-array! writer
- (t2/reducible-select [:model/Table :t.id :t.db_id :t.name :t.schema :t.description]
- {:from [[:metabase_table :t]]
- :join [[:metabase_database :d] [:= :t.db_id :d.id]]
- :where [:and db-filter t-filter]})
- format-table-metadata)
- (.write writer ",\"fields\":")
- (write-json-array! writer
- (t2/reducible-select [:model/Field :f.id :f.table_id :f.parent_id :f.fk_target_field_id
- :f.name :f.description :f.base_type :f.database_type
- :f.effective_type :f.semantic_type :f.coercion_strategy]
- {:from [[:metabase_field :f]]
- :join [[:metabase_table :t] [:= :f.table_id :t.id]
- [:metabase_database :d] [:= :t.db_id :d.id]]
- :where [:and db-filter t-filter f-filter]})
- format-field-metadata)
- (.write writer "}")
- (.flush writer)))
-
-(mr/def ::database-info
- [:map
- [:id ::lib.schema.id/database]
- [:name :string]
- [:engine :string]])
-
-(mr/def ::table-info
- [:map
- [:id ::lib.schema.id/table]
- [:db_id ::lib.schema.id/database]
- [:name :string]
- [:schema {:optional true} :string]
- [:description {:optional true} :string]])
-
-(mr/def ::field-info
- [:map
- [:id ::lib.schema.id/field]
- [:table_id ::lib.schema.id/table]
- [:name :string]
- [:parent_id {:optional true} ::lib.schema.id/field]
- [:fk_target_field_id {:optional true} ::lib.schema.id/field]
- [:description {:optional true} :string]
- [:base_type :string]
- [:database_type {:optional true} :string]
- [:effective_type {:optional true} :string]
- [:semantic_type {:optional true} :string]
- [:coercion_strategy {:optional true} :string]])
-
-(mr/def ::databases-metadata-response
- [:map
- [:databases [:sequential ::database-info]]
- [:tables [:sequential ::table-info]]
- [:fields [:sequential ::field-info]]])
-
-(api.macros/defendpoint :get "/metadata"
- :- (server.streaming-response/streaming-response-schema ::databases-metadata-response)
- "Get metadata (databases, tables, and fields) for all databases visible to the current user.
- Returns a flat structure with three arrays: databases, tables, and fields.
- Response is streamed for efficiency with large schemas.
-
- Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or
- `Query builder and native`) permissions on each database and table."
- []
- (streaming-response {:content-type "application/json; charset=utf-8"} [os _]
- (write-databases-metadata! os)))
-
-;;; --------------------------------------- GET /api/database/field-values ---------------------------------------
-
-(defn- format-field-values-entry
- "Formats a FieldValues row for the /field-values response. Omits `human_readable_values`
- when empty to keep the common case compact."
- [{:keys [field_id values human_readable_values has_more_values]}]
- (m/assoc-some {:field_id field_id
- :values (or values [])
- :has_more_values (boolean has_more_values)}
- :human_readable_values (not-empty human_readable_values)))
-
-(defn- write-field-values!
- "Streams the `field_values` JSON array to `os`. Exports only unconstrained (`:full`)
- FieldValues — sandboxed, impersonation, and linked-filter variants are user-specific
- and excluded from the bulk export. Visibility filter matches `GET /api/database/metadata`
- so every streamed row has a corresponding field entry there."
- [^java.io.OutputStream os]
- (let [db-filter (visible-db-where)
- t-filter (visible-table-where)
- f-filter (visible-field-where)
- fv-filter [:and
- [:= :fv.type "full"]
- [:= :fv.hash_key nil]]
- writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
- (.write writer "{\"field_values\":")
- (write-json-array! writer
- (t2/reducible-select [:model/FieldValues
- :fv.field_id :fv.values :fv.human_readable_values :fv.has_more_values]
- {:from [[:metabase_fieldvalues :fv]]
- :join [[:metabase_field :f] [:= :fv.field_id :f.id]
- [:metabase_table :t] [:= :f.table_id :t.id]
- [:metabase_database :d] [:= :t.db_id :d.id]]
- :where [:and db-filter t-filter f-filter fv-filter]})
- format-field-values-entry)
- (.write writer "}")
- (.flush writer)))
-
-(mr/def ::field-values-info
- [:map
- [:field_id ::lib.schema.id/field]
- [:values [:sequential [:sequential :any]]]
- [:has_more_values :boolean]
- [:human_readable_values {:optional true} [:sequential [:maybe :string]]]])
-
-(mr/def ::field-values-response
- [:map
- [:field_values [:sequential ::field-values-info]]])
-
-(api.macros/defendpoint :get "/field-values"
- :- (server.streaming-response/streaming-response-schema ::field-values-response)
- "Get sampled field values for every field in the instance, streamed as a single
- `{\"field_values\": [...]}` document. Each entry carries `field_id`, `values`,
- optional `human_readable_values`, and `has_more_values`.
-
- Only unconstrained (`:full`) FieldValues are included — sandboxed, impersonation, and
- linked-filter variants are user-specific and would bypass their own enforcement
- mechanisms in a bulk export. Pair with `GET /api/database/metadata` to resolve
- `field_id` to table and field names. Response is streamed for efficiency with large
- schemas.
-
- Admin-only: this endpoint exposes cached values computed over the unrestricted
- dataset, so it would leak data past sandbox / impersonation rules if served to
- regular users."
- []
- (api/check-superuser)
- (streaming-response {:content-type "application/json; charset=utf-8"} [os _]
- (write-field-values! os)))
-
;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------
;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbers
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 1c1d802424f..e4b3ef46c9d 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -749,131 +749,6 @@
(is (true? @connections-stay-open?))
(tx/destroy-db! driver/*driver* empty-dbdef)))))
-(deftest databases-metadata-test
- (testing "GET /api/database/metadata"
- (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"
- :description "A test table"}
- :model/Field {f1-id :id} {:table_id t-id :name "id" :base_type :type/Integer
- :database_type "BIGINT"
- :semantic_type :type/PK}
- :model/Field {f2-id :id} {:table_id t-id :name "created_at" :base_type :type/Text
- :database_type "TIMESTAMP"
- :effective_type :type/DateTime
- :semantic_type :type/Name
- :coercion_strategy :Coercion/ISO8601->DateTime
- :description "The creation time"}
- :model/Field {f3-id :id} {:table_id t-id :name "parent_id" :base_type :type/Integer
- :database_type "BIGINT"
- :semantic_type :type/FK
- :fk_target_field_id f1-id}]
- (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202 "database/metadata")]
- (is (=? {:id db-id :name "test-db" :engine "h2"}
- (m/find-first (comp #{db-id} :id) databases)))
- (is (=? {:id t-id :db_id db-id :name "my_table" :schema "PUBLIC" :description "A test table"}
- (m/find-first (comp #{t-id} :id) tables)))
- (is (=? {:id f1-id :table_id t-id :name "id" :base_type "type/Integer" :database_type "BIGINT"
- :semantic_type "type/PK"}
- (m/find-first (comp #{f1-id} :id) fields)))
- (is (=? {:id f2-id
- :table_id t-id
- :name "created_at"
- :base_type "type/Text"
- :database_type "TIMESTAMP"
- :effective_type "type/DateTime"
- :semantic_type "type/Name"
- :coercion_strategy "Coercion/ISO8601->DateTime"
- :description "The creation time"}
- (m/find-first (comp #{f2-id} :id) fields)))
- (is (=? {:id f3-id
- :table_id t-id
- :name "parent_id"
- :base_type "type/Integer"
- :database_type "BIGINT"
- :semantic_type "type/FK"
- :fk_target_field_id f1-id}
- (m/find-first (comp #{f3-id} :id) fields)))))))
-
-(deftest databases-metadata-no-perms-test
- (testing "GET /api/database/metadata — user without data perms sees nothing"
- (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"}
- :model/Field _ {:table_id t-id :name "id" :base_type :type/Integer
- :database_type "BIGINT"}]
- (mt/with-no-data-perms-for-all-users!
- (is (= {:databases [] :tables [] :fields []}
- (mt/user-http-request :rasta :get 202 "database/metadata")))))))
-
-(deftest databases-field-values-test
- (testing "GET /api/database/field-values"
- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
- :model/Field {f1-id :id} {:table_id t-id :name "state" :base_type :type/Text
- :database_type "VARCHAR"}
- :model/Field {f2-id :id} {:table_id t-id :name "rating" :base_type :type/Integer
- :database_type "INTEGER"}
- :model/FieldValues _ {:field_id f1-id :type :full
- :values [["CA"] ["NY"] ["TX"]]
- :has_more_values false}
- :model/FieldValues _ {:field_id f2-id :type :full
- :values [[1] [2] [3]]
- :human_readable_values ["Low" "Mid" "High"]
- :has_more_values true}]
- (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")
- by-field (into {} (map (juxt :field_id identity)) field_values)]
- (is (=? {:field_id f1-id
- :values [["CA"] ["NY"] ["TX"]]
- :has_more_values false}
- (by-field f1-id)))
- (is (nil? (:human_readable_values (by-field f1-id)))
- "human_readable_values is omitted when empty")
- (is (=? {:field_id f2-id
- :values [[1] [2] [3]]
- :human_readable_values ["Low" "Mid" "High"]
- :has_more_values true}
- (by-field f2-id)))))))
-
-(deftest databases-field-values-non-admin-test
- (testing "GET /api/database/field-values — non-admin requests are rejected"
- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
- :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text
- :database_type "VARCHAR"}
- :model/FieldValues _ {:field_id f-id :type :full
- :values [["CA"] ["NY"]]
- :has_more_values false}]
- (is (= "You don't have permissions to do that."
- (mt/user-http-request :rasta :get 403 "database/field-values"))))))
-
-(deftest databases-field-values-excludes-non-full-test
- (testing "GET /api/database/field-values — only :full FieldValues are included"
- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}
- :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text
- :database_type "VARCHAR"}
- :model/FieldValues _ {:field_id f-id :type :full
- :values [["CA"]]
- :has_more_values false}
- :model/FieldValues _ {:field_id f-id :type :sandbox
- :hash_key "sandbox-hash"
- :values [["NY"]]
- :has_more_values false}]
- (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")
- for-field (filter #(= f-id (:field_id %)) field_values)]
- (is (= 1 (count for-field))
- "only the :full entry streams; :sandbox / other variants are excluded")
- (is (= [["CA"]] (-> for-field first :values)))))))
-
-(deftest databases-metadata-excludes-audit-db-test
- (testing "GET /api/database/metadata — audit (internal) database, its tables, and its fields are excluded"
- (mt/with-temp [:model/Database {db-id :id} {:name "audit-db" :engine :h2 :is_audit true}
- :model/Table {t-id :id} {:db_id db-id :name "audit_table" :schema "PUBLIC"}
- :model/Field {f-id :id} {:table_id t-id :name "audit_col" :base_type :type/Integer}]
- (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202 "database/metadata")]
- (is (nil? (m/find-first (comp #{db-id} :id) databases)))
- (is (nil? (m/find-first (comp #{t-id} :id) tables)))
- (is (nil? (m/find-first (comp #{f-id} :id) fields)))))))
-
(deftest ^:parallel fetch-database-metadata-test
(testing "GET /api/database/:id/metadata"
(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)