The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
External metadata (#73612)
Alexander Polyankin72975d87c3db
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 4f4bb54cbe2..69f85e4a6f4 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2588,7 +2588,6 @@requestsample-datasecrets- serversettingssyncupload@@ -2600,7 +2599,6 @@:model/Collection:model/Database:model/Field- :model/FieldValues:model/Table}}xrays@@ -3303,6 +3301,7 @@libloggermodels+ permissionspluginspremium-featuressearchdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex a51cb93d7e3..32b3e195a77 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -3,6 +3,8 @@[clojure.java.io :as io][clojure.string :as str][java-time.api :as t]+ [metabase-enterprise.serialization.metadata :as metadata]+ [metabase-enterprise.serialization.schema :as schema][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase-enterprise.serialization.v2.load :as v2.load]@@ -20,6 +22,7 @@[metabase.util :as u][metabase.util.compress :as u.compress][metabase.util.date-2 :as u.date]+ [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -306,6 +309,89 @@(finally(io/delete-file (:tempfile file)))))+;;; ----------------------------------- GET /api/ee/serialization/metadata/export -----------------------------------++(defn- write-json-array!+ "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded+ directly with no transformation — apply per-row formatting via an `eduction` (or other+ transducer pipeline) before passing in.++ `run!` is required here because it dispatches through `reduce`, which consumes the+ `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.+ `doseq` cannot be used: it walks a seq, and producing a seq from the reducible+ would realize every row into memory — defeating the point of streaming."+ [^java.io.Writer writer reducible]+ (.write writer "[")+ (let [first? (volatile! true)]+ (run! (fn [row]+ (if @first?+ (vreset! first? false)+ (.write writer ","))+ (json/encode-to row writer {}))+ reducible))+ (.write writer "]"))++(defn- write-json-object!+ "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible+ of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)+ of already-formatted values to encode.++ `run!` is used over the entries — `doseq` would walk a seq and realize the underlying+ reducible, defeating streaming."+ [^java.io.Writer writer entries]+ (.write writer "{")+ (let [first? (volatile! true)]+ (run! (fn [[entry-name objects]]+ (if @first? (vreset! first? false) (.write writer ","))+ (.write writer (str "\"" entry-name "\":"))+ (write-json-array! writer objects))+ entries))+ (.write writer "}"))++(defn- write-databases-metadata!+ "Streams the databases/tables/fields metadata to the given OutputStream. Sections are+ included only when their `with-...?` flag is true. All references — database, table,+ fk_target_field — are emitted in serdes-portable form (names rather than numeric IDs)+ so the response can be ingested by another Metabase instance with different surrogate+ keys.++ Warehouses with large schemas can produce gigabytes of metadata, so rows are pulled+ from reducible queries and streamed directly to the writer — memory stays bounded+ regardless of schema size."+ [^java.io.OutputStream os {:keys [with-databases? with-tables? with-fields?]}]+ (let [writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]+ (write-json-object!+ writer+ (cond-> []+ with-databases? (conj ["databases" (metadata/reducible-databases)])+ with-tables? (conj ["tables" (metadata/reducible-tables)])+ with-fields? (conj ["fields" (metadata/reducible-fields)])))+ (.flush writer)))++(api.macros/defendpoint :get "/metadata/export"+ :- (sr/streaming-response-schema ::schema/metadata-export-response)+ "Get warehouse metadata (databases, tables, and fields) for all databases visible to the+ current user, with all references emitted in serdes-portable form (database names, table+ `[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).++ Sections must be opted into with the `with-databases`, `with-tables`, and `with-fields`+ query parameters — they all default to `false`. The response is streamed for efficiency+ with large schemas.++ Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or+ `Query builder and native`) permissions on each database and table."+ [_route-params+ {:keys [with-databases with-tables with-fields]}+ :- [:map+ [:with-databases {:default false} [:maybe :boolean]]+ [:with-tables {:default false} [:maybe :boolean]]+ [:with-fields {:default false} [:maybe :boolean]]]]+ (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]+ (write-databases-metadata! os+ {:with-databases? with-databases+ :with-tables? with-tables+ :with-fields? with-fields})))+(def ^{:arglists '([request respond raise])} routes"`/api/ee/serialization` routes."(api.macros/ns-handler *ns* +auth))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata.cljnew file mode 100644index 00000000000..22e459d4e16--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj@@ -0,0 +1,187 @@+(ns metabase-enterprise.serialization.metadata+ "Reducible queries and row formatters that produce serdes-portable warehouse metadata+ (databases, tables, and fields) for the `/api/ee/serialization/metadata/export` endpoint.+ All references — database, table, fk_target_field — are emitted as portable references+ (names rather than numeric IDs) so the response can be ingested by another Metabase+ instance with different surrogate keys."+ (:require+ [medley.core :as m]+ [metabase.api.common :as api]+ [metabase.permissions.core :as perms]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(defn- perm-user-info+ "Current-user info passed to `perms/visible-*-filter-select` to scope the visibility filters."+ []+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*})++(defn- perm-mapping+ "Permission mapping required to include a database/table in the export."+ []+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})++(defn- visible-db-where+ "Honeysql predicate selecting non-audit, non-router databases visible to the current user.+ `d` is the table alias of the joined `metabase_database`."+ [d]+ [:and+ [:= (u/qualified-key d :is_audit) false]+ [:= (u/qualified-key d :router_database_id) nil]+ [:in (u/qualified-key d :id) (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])++(defn- visible-table-where+ "Honeysql predicate selecting active, non-hidden tables visible to the current user.+ `t` is the table alias of the joined `metabase_table`."+ [t]+ [:and+ [:= (u/qualified-key t :active) true]+ [:= (u/qualified-key t :visibility_type) nil]+ [:in (u/qualified-key t :id) (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])++(defn- visible-field-where+ "Honeysql predicate selecting active, non-sensitive fields. `f` is the table alias of the+ joined `metabase_field`."+ [f]+ [:and+ [:= (u/qualified-key f :active) true]+ [:<> (u/qualified-key f :visibility_type) "sensitive"]])++(defn- reducible-databases-query+ "Raw reducible-query streaming visible-database rows for [[reducible-databases]]."+ []+ (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]+ :from [[:metabase_database :db]]+ :where (visible-db-where :db)}))++(defn- reducible-tables-query+ "Raw reducible-query streaming visible-table rows for [[reducible-tables]]."+ []+ (t2/reducible-query {:select [[:db.name :db_name]+ [:table.schema :schema]+ [:table.name :table_name]+ [:table.description :description]]+ :from [[:metabase_table :table]]+ :join [[:metabase_database :db] [:= :table.db_id :db.id]]+ :where [:and (visible-db-where :db) (visible-table-where :table)]}))++(defn- reducible-fields-query+ "Raw reducible-query streaming visible-field rows for [[reducible-fields]]. Visibility+ filters on the FK target chain are folded into the LEFT JOIN ON clauses (rather than the+ WHERE) so that an inaccessible target fails the join and the `fk_*` columns come back as+ NULL — [[format-field-row]] then drops the `:fk_target_field_id` for that row."+ []+ (t2/reducible-query+ {:select [[:db.name :db_name]+ [:table.schema :table_schema]+ [:table.name :table_name]+ [:field.name :field_name]+ [:field.description :description]+ [:field.base_type :base_type]+ [:field.database_type :database_type]+ [:field.effective_type :effective_type]+ [:field.semantic_type :semantic_type]+ [:field.coercion_strategy :coercion_strategy]+ [:field.nfc_path :nfc_path]+ [:fk_db.name :fk_db_name]+ [:fk_table.schema :fk_table_schema]+ [:fk_table.name :fk_table_name]+ [:fk_field.name :fk_field_name]+ [:fk_field.nfc_path :fk_field_nfc_path]]+ :from [[:metabase_field :field]]+ :join [[:metabase_table :table] [:= :field.table_id :table.id]+ [:metabase_database :db] [:= :table.db_id :db.id]]+ :left-join [[:metabase_field :fk_field] [:and [:= :field.fk_target_field_id :fk_field.id] (visible-field-where :fk_field)]+ [:metabase_table :fk_table] [:and [:= :fk_field.table_id :fk_table.id] (visible-table-where :fk_table)]+ [:metabase_database :fk_db] [:and [:= :fk_table.db_id :fk_db.id] (visible-db-where :fk_db)]]+ :where [:and (visible-db-where :db) (visible-table-where :table) (visible-field-where :field)]}))++(defn- decode-nfc-path+ "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for+ null/empty, the original value when it's already a sequential collection, or the decoded+ vector when it's a JSON string."+ [nfc-path]+ (cond+ (nil? nfc-path) nil+ (sequential? nfc-path) (seq nfc-path)+ (string? nfc-path) (seq (json/decode nfc-path))))++(defn- external-database-id+ "Portable id for a database — its name."+ [db-name]+ db-name)++(defn- external-table-id+ "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."+ [db-name schema table-name]+ [db-name schema table-name])++(defn- external-field-id+ "Portable id for a field. When `nfc-path` is set (JSON-nested column), the path replaces+ the field's display name; the field's own name is the leaf of `nfc-path` joined with+ arrows, so the path is the canonical structural representation."+ [db-name schema table-name field-name nfc-path]+ (if (seq nfc-path)+ (into [db-name schema table-name] nfc-path)+ [db-name schema table-name field-name]))++(defn format-database-row+ "JSON shape for one database row, identifying the database by its portable id."+ [{:keys [name engine]}]+ {:id (external-database-id name) :name name :engine engine})++(defn format-table-row+ "JSON shape for one table row, with portable references for `:id` and `:db_id`. Optional+ fields are omitted when nil."+ [{:keys [db_name schema table_name description]}]+ (m/assoc-some {:id (external-table-id db_name schema table_name)+ :db_id (external-database-id db_name)+ :name table_name}+ :schema schema+ :description description))++(defn format-field-row+ "JSON shape for one field row, with portable references for `:id`, `:table_id`, `:parent_id`,+ and `:fk_target_field_id`. `:parent_id` is derived from `nfc_path` (the portable id with the+ last path element dropped) and is only emitted when `nfc_path` has at least two elements.+ Optional fields are omitted when nil."+ [{:keys [db_name table_schema table_name field_name description base_type database_type+ effective_type semantic_type coercion_strategy nfc_path+ fk_db_name fk_table_schema fk_table_name fk_field_name fk_field_nfc_path]}]+ (let [nfc-path (decode-nfc-path nfc_path)+ parent-id (when-some [parent-nfc (seq (butlast nfc-path))]+ (external-field-id db_name table_schema table_name nil parent-nfc))+ fk-field-nfc (decode-nfc-path fk_field_nfc_path)+ fk-target-id (when (and fk_db_name fk_table_name fk_field_name)+ (external-field-id fk_db_name fk_table_schema fk_table_name+ fk_field_name fk-field-nfc))]+ (m/assoc-some {:id (external-field-id db_name table_schema table_name field_name nfc-path)+ :table_id (external-table-id db_name table_schema table_name)+ :name field_name}+ :description description+ :base_type base_type+ :database_type database_type+ :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)+ :semantic_type semantic_type+ :coercion_strategy coercion_strategy+ :nfc_path nfc-path+ :parent_id parent-id+ :fk_target_field_id fk-target-id)))++(defn reducible-databases+ "Eduction streaming visible databases as already-formatted JSON-shaped rows."+ []+ (eduction (map format-database-row) (reducible-databases-query)))++(defn reducible-tables+ "Eduction streaming visible tables as already-formatted JSON-shaped rows."+ []+ (eduction (map format-table-row) (reducible-tables-query)))++(defn reducible-fields+ "Eduction streaming visible fields as already-formatted JSON-shaped rows."+ []+ (eduction (map format-field-row) (reducible-fields-query)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.cljnew file mode 100644index 00000000000..740ec109256--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj@@ -0,0 +1,49 @@+(ns metabase-enterprise.serialization.schema+ "Malli schemas for the serdes-portable shapes produced by+ [[metabase-enterprise.serialization.metadata]]."+ (:require+ [metabase.util.malli.registry :as mr]))++(mr/def ::external-database-id+ :string)++(mr/def ::external-table-id+ [:tuple :string [:maybe :string] :string])++(mr/def ::external-field-id+ [:cat :string [:maybe :string] :string [:+ :string]])++(mr/def ::database-info+ [:map+ [:id ::external-database-id]+ [:name :string]+ [:engine :string]])++(mr/def ::table-info+ [:map+ [:id ::external-table-id]+ [:db_id ::external-database-id]+ [:name :string]+ [:schema {:optional true} :string]+ [:description {:optional true} :string]])++(mr/def ::field-info+ [:map+ [:id ::external-field-id]+ [:table_id ::external-table-id]+ [:name :string]+ [:parent_id {:optional true} ::external-field-id]+ [:fk_target_field_id {:optional true} ::external-field-id]+ [:description {:optional true} :string]+ [:base_type :string]+ [:database_type {:optional true} :string]+ [:effective_type {:optional true} :string]+ [:semantic_type {:optional true} :string]+ [:coercion_strategy {:optional true} :string]+ [:nfc_path {:optional true} [:sequential :string]]])++(mr/def ::metadata-export-response+ [:map+ [:databases {:optional true} [:sequential ::database-info]]+ [:tables {:optional true} [:sequential ::table-info]]+ [:fields {:optional true} [:sequential ::field-info]]])diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 884da4a7a22..56a212ef83a 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -4,10 +4,12 @@[clojure.string :as str][clojure.test :refer :all][clojure.walk :as walk]+ [medley.core :as m][metabase-enterprise.serialization.api :as api.serialization][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase.analytics.snowplow-test :as snowplow-test][metabase.models.serialization :as serdes]+ [metabase.permissions.core :as perms][metabase.search.core :as search][metabase.search.test-util :as search.tu][metabase.test :as mt]@@ -441,6 +443,289 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file (byte-array 0)})))))))+(deftest metadata-export-basic-test+ (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"+ :description "A test table"}+ :model/Field {f-name :name} {:table_id t-id+ :base_type :type/Integer+ :database_type "BIGINT"+ :semantic_type :type/PK}]+ (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-databases true+ :with-tables true+ :with-fields true)+ test-db (m/find-first (comp #{db-name} :id) databases)+ test-table (m/find-first (comp #{[db-name "PUBLIC" t-name]} :id) tables)+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]+ (is (=? {:id db-name :name db-name :engine "h2"}+ test-db))+ (is (=? {:id [db-name "PUBLIC" t-name]+ :db_id db-name+ :name t-name+ :schema "PUBLIC"+ :description "A test table"}+ test-table))+ (is (=? {:id [db-name "PUBLIC" t-name f-name]+ :table_id [db-name "PUBLIC" t-name]+ :name f-name+ :base_type "type/Integer"+ :database_type "BIGINT"+ :semantic_type "type/PK"}+ test-field)))))))++(deftest metadata-export-optional-properties-test+ (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {f-name :name} {:table_id t-id+ :base_type :type/Text+ :database_type "VARCHAR"+ :effective_type :type/DateTime+ :coercion_strategy :Coercion/ISO8601->DateTime+ :description "When this happened"}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]+ (is (=? {:id [db-name "PUBLIC" t-name f-name]+ :name f-name+ :base_type "type/Text"+ :effective_type "type/DateTime"+ :coercion_strategy "Coercion/ISO8601->DateTime"+ :description "When this happened"}+ test-field)))))))++(deftest metadata-export-parent-field-test+ (testing "GET /api/ee/serialization/metadata/export — parent_id is nfc_path with the last element dropped, omitted for root fields"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {root-name :name} {:table_id t-id+ :base_type :type/Text}+ :model/Field _ {:table_id t-id+ :base_type :type/Text+ :nfc_path ["data" "city"]}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name root-name]} :id) fields)+ test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name "data" "city"]} :id) fields)]+ (is (=? {:id [db-name "PUBLIC" t-name root-name]} test-field-1))+ (is (not (contains? test-field-1 :parent_id)))+ (is (=? {:id [db-name "PUBLIC" t-name "data" "city"]+ :parent_id [db-name "PUBLIC" t-name "data"]}+ test-field-2)))))))++(deftest metadata-export-fk-test+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted in portable form"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {pk-id :id pk-name :name} {:table_id t-id+ :base_type :type/Integer+ :semantic_type :type/PK}+ :model/Field {fk-name :name} {:table_id t-id+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id pk-id}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name pk-name]} :id) fields)+ test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]+ (is (=? {:id [db-name "PUBLIC" t-name pk-name] :semantic_type "type/PK"}+ test-field-1))+ (is (=? {:id [db-name "PUBLIC" t-name fk-name]+ :table_id [db-name "PUBLIC" t-name]+ :semantic_type "type/FK"+ :fk_target_field_id (:id test-field-1)}+ test-field-2)))))))++(deftest metadata-export-hidden-table-test+ (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Table {hidden-id :id hidden-t-name :name} {:db_id db-id :schema "PUBLIC"+ :visibility_type :hidden}+ :model/Field {visible-f-name :name} {:table_id visible-id+ :base_type :type/Integer}+ :model/Field {hidden-f-name :name} {:table_id hidden-id+ :base_type :type/Integer}]+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-tables true :with-fields true)]+ (is (some (comp #{visible-t-name} :name) tables))+ (is (not-any? (comp #{hidden-t-name} :name) tables))+ (is (some (comp #{visible-f-name} :name) fields))+ (is (not-any? (comp #{hidden-f-name} :name) fields)))))))++(deftest metadata-export-hidden-table-fk-test+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is hidden"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"+ :visibility_type :hidden}+ :model/Field {target-id :id} {:table_id hidden-id+ :base_type :type/Integer+ :semantic_type :type/PK}+ :model/Field {fk-name :name} {:table_id visible-id+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id target-id}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field (m/find-first (comp #{[db-name "PUBLIC" v-name fk-name]} :id) fields)]+ (is (some? test-field))+ (is (not (contains? test-field :fk_target_field_id))))))))++(deftest metadata-export-inactive-table-test+ (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"+ :active false}+ :model/Field {active-f-name :name} {:table_id t-id+ :base_type :type/Integer}+ :model/Field {inactive-f-name :name} {:table_id t-id+ :base_type :type/Integer+ :active false}]+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-tables true :with-fields true)]+ (is (not-any? (comp #{t-name} :name) tables))+ (is (not-any? (comp #{active-f-name} :name) fields))+ (is (not-any? (comp #{inactive-f-name} :name) fields)))))))++(deftest metadata-export-sensitive-field-test+ (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {email-name :name} {:table_id t-id :base_type :type/Text}+ :model/Field {secret-name :name} {:table_id t-id :base_type :type/Text+ :visibility_type :sensitive}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)]+ (is (some (comp #{email-name} :name) fields))+ (is (not-any? (comp #{secret-name} :name) fields)))))))++(deftest metadata-export-sensitive-fk-test+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target field is sensitive"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {target-id :id} {:table_id t-id+ :base_type :type/Text+ :visibility_type :sensitive}+ :model/Field {fk-name :name} {:table_id t-id+ :base_type :type/Text+ :semantic_type :type/FK+ :fk_target_field_id target-id}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]+ (is (some? test-field))+ (is (not (contains? test-field :fk_target_field_id))))))))++(deftest metadata-export-inactive-field-test+ (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {active-f-name :name} {:table_id t-id+ :base_type :type/Integer}+ :model/Field {inactive-f-name :name} {:table_id t-id+ :base_type :type/Integer+ :active false}]+ (let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-tables true :with-fields true)]+ (is (some (comp #{t-name} :name) tables))+ (is (some (comp #{active-f-name} :name) fields))+ (is (not-any? (comp #{inactive-f-name} :name) fields)))))))++(deftest metadata-export-db-routing-test+ (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database _ {:engine :h2}+ :model/Database {primary-id :id primary-name :name} {:engine :h2}+ :model/Database {mirror-name :name} {:engine :h2+ :router_database_id primary-id}]+ (let [{:keys [databases]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export"+ :with-databases true)]+ (is (some (comp #{primary-name} :name) databases))+ (is (not-any? (comp #{mirror-name} :name) databases)))))))++(deftest metadata-export-table-permission-test+ (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Table {restricted-id :id restricted-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Field {ok-f-name :name} {:table_id accessible-id+ :base_type :type/Integer}+ :model/Field {no-f-name :name} {:table_id restricted-id+ :base_type :type/Integer}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (perms/set-database-permission! pg db-id :perms/create-queries :no)+ (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)+ (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)+ (let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202+ "ee/serialization/metadata/export"+ :with-tables true :with-fields true)]+ (is (some (comp #{accessible-name} :name) tables))+ (is (not-any? (comp #{restricted-name} :name) tables))+ (is (some (comp #{ok-f-name} :name) fields))+ (is (not-any? (comp #{no-f-name} :name) fields)))))))++(deftest metadata-export-table-permission-fk-test+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is not accessible"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}+ :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}+ :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {target-id :id} {:table_id restricted-id+ :base_type :type/Integer+ :semantic_type :type/PK}+ :model/Field {fk-name :name} {:table_id accessible-id+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id target-id}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (perms/set-database-permission! pg db-id :perms/create-queries :no)+ (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)+ (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)+ (let [{:keys [fields]} (mt/user-http-request :rasta :get 202+ "ee/serialization/metadata/export"+ :with-fields true)+ test-field (m/find-first (comp #{[db-name "PUBLIC" accessible-name fk-name]} :id) fields)]+ (is (some? test-field))+ (is (not (contains? test-field :fk_target_field_id))))))))++(deftest metadata-export-token-feature-test+ (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"+ (mt/with-premium-features #{}+ (mt/assert-has-premium-feature-error+ "Serialization"+ (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))+(deftest serialization-cleanup-test(testing "No temp files are left behind after export/import operations"(let [known-files (set (.list (io/file api.serialization/parent-dir)))]diff --git a/src/metabase/util.cljc b/src/metabase/util.cljcindex a1a144a6ab3..d95da88151f 100644--- a/src/metabase/util.cljc+++ b/src/metabase/util.cljc@@ -57,6 +57,7 @@format-nanosecondsformat-secondsformat-plural+ qualified-keyqualified-name])#?(:clj (p/import-vars [u.jvmdiff --git a/src/metabase/util/format.cljc b/src/metabase/util/format.cljcindex 60be7ea69bd..aa675ed8730 100644--- a/src/metabase/util/format.cljc+++ b/src/metabase/util/format.cljc@@ -103,6 +103,11 @@plural(str singular \s)))))+(defn qualified-key+ "(u/qualified-key :t :id) -> :t.id"+ [a b]+ (keyword (str (name a) "." (name b))))+(defn qualified-name"Return `k` as a string, qualified by its namespace, if any (unlike `name`). Handles `nil` values gracefully as well(also unlike `name`).diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex b30943d807d..d7d98941f89 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -28,7 +28,6 @@[metabase.request.core :as request][metabase.sample-data.core :as sample-data][metabase.secrets.core :as secret]- [metabase.server.streaming-response :as server.streaming-response :refer [streaming-response]][metabase.settings.core :as setting][metabase.sync.core :as sync][metabase.sync.schedules :as sync.schedules]@@ -38,7 +37,6 @@[metabase.util.cron :as u.cron][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :as i18n :refer [deferred-tru trs tru]]- [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]@@ -546,243 +544,6 @@:when query][query model])})))-;;; ----------------------------------------- GET /api/database/metadata --------------------------------------------(defn- format-database-metadata- "Formats a Database record for the /metadata endpoint response."- [{:keys [id name engine]}]- {:id id :name name :engine engine})--(defn- format-table-metadata- "Formats a Table record for the /metadata endpoint response, omitting nil optional fields."- [{:keys [id db_id name schema description]}]- (m/assoc-some {:id id :db_id db_id :name name}- :schema schema- :description description))--(defn- format-field-metadata- "Formats a Field record for the /metadata endpoint response. Includes effective_type only when it differs from base_type."- [{:keys [id table_id parent_id fk_target_field_id name description base_type database_type effective_type semantic_type coercion_strategy]}]- (m/assoc-some {:id id :table_id table_id :name name}- :parent_id parent_id- :fk_target_field_id fk_target_field_id- :description description- :base_type base_type- :database_type database_type- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)- :semantic_type semantic_type- :coercion_strategy coercion_strategy))--(defn- perm-user-info- "User information used to check permissions."- []- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*})--(defn- perm-mapping- "Permission mapping used to filter databases and tables to those visible to the current user.- Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or `Query builder and native`)."- []- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})--(defn- write-json-array!- "Streams a reducible collection as a JSON array to a Writer, applying `format-fn` to each row.-- `run!` is required here because it dispatches through `reduce`, which consumes the- `IReduceInit` returned by `t2/reducible-select` row-by-row without materializing.- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible- would realize every row into memory — defeating the point of streaming."- [^java.io.Writer writer reducible format-fn]- (.write writer "[")- (let [first? (volatile! true)]- (run! (fn [row]- (if @first?- (vreset! first? false)- (.write writer ","))- (json/encode-to (format-fn row) writer {}))- reducible))- (.write writer "]"))--(defn- visible-db-where- "Honeysql `:where` clause restricting a `:metabase_database :d` join to databases that- belong in a bulk export for the current user: not audit, not a router DB, and inside- the query-builder visibility filter."- []- [:and- [:= :d.is_audit false]- [:= :d.router_database_id nil]- [:in :d.id (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])--(defn- visible-table-where- "Honeysql `:where` clause restricting a `:metabase_table :t` join to active, non-hidden- tables inside the current user's query-builder visibility filter."- []- [:and- [:= :t.active true]- [:= :t.visibility_type nil]- [:in :t.id (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])--(defn- visible-field-where- "Honeysql `:where` clause restricting a `:metabase_field :f` join to active,- non-sensitive fields."- []- [:and- [:= :f.active true]- [:<> :f.visibility_type "sensitive"]])--(defn- write-databases-metadata!- "Streams the databases/tables/fields metadata as JSON to the given OutputStream.-- Warehouses with large schemas can produce gigabytes of metadata, so streaming is- required — materializing the full response in memory would OOM the server. Each- section is written directly to the underlying writer as rows are pulled from a- reducible query, keeping memory usage bounded regardless of schema size."- [^java.io.OutputStream os]- (let [db-filter (visible-db-where)- t-filter (visible-table-where)- f-filter (visible-field-where)- writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]- (.write writer "{\"databases\":")- (write-json-array! writer- (t2/reducible-select [:model/Database :d.id :d.name :d.engine]- {:from [[:metabase_database :d]]- :where db-filter})- format-database-metadata)- (.write writer ",\"tables\":")- (write-json-array! writer- (t2/reducible-select [:model/Table :t.id :t.db_id :t.name :t.schema :t.description]- {:from [[:metabase_table :t]]- :join [[:metabase_database :d] [:= :t.db_id :d.id]]- :where [:and db-filter t-filter]})- format-table-metadata)- (.write writer ",\"fields\":")- (write-json-array! writer- (t2/reducible-select [:model/Field :f.id :f.table_id :f.parent_id :f.fk_target_field_id- :f.name :f.description :f.base_type :f.database_type- :f.effective_type :f.semantic_type :f.coercion_strategy]- {:from [[:metabase_field :f]]- :join [[:metabase_table :t] [:= :f.table_id :t.id]- [:metabase_database :d] [:= :t.db_id :d.id]]- :where [:and db-filter t-filter f-filter]})- format-field-metadata)- (.write writer "}")- (.flush writer)))--(mr/def ::database-info- [:map- [:id ::lib.schema.id/database]- [:name :string]- [:engine :string]])--(mr/def ::table-info- [:map- [:id ::lib.schema.id/table]- [:db_id ::lib.schema.id/database]- [:name :string]- [:schema {:optional true} :string]- [:description {:optional true} :string]])--(mr/def ::field-info- [:map- [:id ::lib.schema.id/field]- [:table_id ::lib.schema.id/table]- [:name :string]- [:parent_id {:optional true} ::lib.schema.id/field]- [:fk_target_field_id {:optional true} ::lib.schema.id/field]- [:description {:optional true} :string]- [:base_type :string]- [:database_type {:optional true} :string]- [:effective_type {:optional true} :string]- [:semantic_type {:optional true} :string]- [:coercion_strategy {:optional true} :string]])--(mr/def ::databases-metadata-response- [:map- [:databases [:sequential ::database-info]]- [:tables [:sequential ::table-info]]- [:fields [:sequential ::field-info]]])--(api.macros/defendpoint :get "/metadata"- :- (server.streaming-response/streaming-response-schema ::databases-metadata-response)- "Get metadata (databases, tables, and fields) for all databases visible to the current user.- Returns a flat structure with three arrays: databases, tables, and fields.- Response is streamed for efficiency with large schemas.-- Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or- `Query builder and native`) permissions on each database and table."- []- (streaming-response {:content-type "application/json; charset=utf-8"} [os _]- (write-databases-metadata! os)))--;;; --------------------------------------- GET /api/database/field-values -----------------------------------------(defn- format-field-values-entry- "Formats a FieldValues row for the /field-values response. Omits `human_readable_values`- when empty to keep the common case compact."- [{:keys [field_id values human_readable_values has_more_values]}]- (m/assoc-some {:field_id field_id- :values (or values [])- :has_more_values (boolean has_more_values)}- :human_readable_values (not-empty human_readable_values)))--(defn- write-field-values!- "Streams the `field_values` JSON array to `os`. Exports only unconstrained (`:full`)- FieldValues — sandboxed, impersonation, and linked-filter variants are user-specific- and excluded from the bulk export. Visibility filter matches `GET /api/database/metadata`- so every streamed row has a corresponding field entry there."- [^java.io.OutputStream os]- (let [db-filter (visible-db-where)- t-filter (visible-table-where)- f-filter (visible-field-where)- fv-filter [:and- [:= :fv.type "full"]- [:= :fv.hash_key nil]]- writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]- (.write writer "{\"field_values\":")- (write-json-array! writer- (t2/reducible-select [:model/FieldValues- :fv.field_id :fv.values :fv.human_readable_values :fv.has_more_values]- {:from [[:metabase_fieldvalues :fv]]- :join [[:metabase_field :f] [:= :fv.field_id :f.id]- [:metabase_table :t] [:= :f.table_id :t.id]- [:metabase_database :d] [:= :t.db_id :d.id]]- :where [:and db-filter t-filter f-filter fv-filter]})- format-field-values-entry)- (.write writer "}")- (.flush writer)))--(mr/def ::field-values-info- [:map- [:field_id ::lib.schema.id/field]- [:values [:sequential [:sequential :any]]]- [:has_more_values :boolean]- [:human_readable_values {:optional true} [:sequential [:maybe :string]]]])--(mr/def ::field-values-response- [:map- [:field_values [:sequential ::field-values-info]]])--(api.macros/defendpoint :get "/field-values"- :- (server.streaming-response/streaming-response-schema ::field-values-response)- "Get sampled field values for every field in the instance, streamed as a single- `{\"field_values\": [...]}` document. Each entry carries `field_id`, `values`,- optional `human_readable_values`, and `has_more_values`.-- Only unconstrained (`:full`) FieldValues are included — sandboxed, impersonation, and- linked-filter variants are user-specific and would bypass their own enforcement- mechanisms in a bulk export. Pair with `GET /api/database/metadata` to resolve- `field_id` to table and field names. Response is streamed for efficiency with large- schemas.-- Admin-only: this endpoint exposes cached values computed over the unrestricted- dataset, so it would leak data past sandbox / impersonation rules if served to- regular users."- []- (api/check-superuser)- (streaming-response {:content-type "application/json; charset=utf-8"} [os _]- (write-field-values! os)))-;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbersdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 1c1d802424f..e4b3ef46c9d 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -749,131 +749,6 @@(is (true? @connections-stay-open?))(tx/destroy-db! driver/*driver* empty-dbdef)))))-(deftest databases-metadata-test- (testing "GET /api/database/metadata"- (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"- :description "A test table"}- :model/Field {f1-id :id} {:table_id t-id :name "id" :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/PK}- :model/Field {f2-id :id} {:table_id t-id :name "created_at" :base_type :type/Text- :database_type "TIMESTAMP"- :effective_type :type/DateTime- :semantic_type :type/Name- :coercion_strategy :Coercion/ISO8601->DateTime- :description "The creation time"}- :model/Field {f3-id :id} {:table_id t-id :name "parent_id" :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/FK- :fk_target_field_id f1-id}]- (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202 "database/metadata")]- (is (=? {:id db-id :name "test-db" :engine "h2"}- (m/find-first (comp #{db-id} :id) databases)))- (is (=? {:id t-id :db_id db-id :name "my_table" :schema "PUBLIC" :description "A test table"}- (m/find-first (comp #{t-id} :id) tables)))- (is (=? {:id f1-id :table_id t-id :name "id" :base_type "type/Integer" :database_type "BIGINT"- :semantic_type "type/PK"}- (m/find-first (comp #{f1-id} :id) fields)))- (is (=? {:id f2-id- :table_id t-id- :name "created_at"- :base_type "type/Text"- :database_type "TIMESTAMP"- :effective_type "type/DateTime"- :semantic_type "type/Name"- :coercion_strategy "Coercion/ISO8601->DateTime"- :description "The creation time"}- (m/find-first (comp #{f2-id} :id) fields)))- (is (=? {:id f3-id- :table_id t-id- :name "parent_id"- :base_type "type/Integer"- :database_type "BIGINT"- :semantic_type "type/FK"- :fk_target_field_id f1-id}- (m/find-first (comp #{f3-id} :id) fields)))))))--(deftest databases-metadata-no-perms-test- (testing "GET /api/database/metadata — user without data perms sees nothing"- (mt/with-temp [:model/Database {db-id :id} {:name "test-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "my_table" :schema "PUBLIC"}- :model/Field _ {:table_id t-id :name "id" :base_type :type/Integer- :database_type "BIGINT"}]- (mt/with-no-data-perms-for-all-users!- (is (= {:databases [] :tables [] :fields []}- (mt/user-http-request :rasta :get 202 "database/metadata")))))))--(deftest databases-field-values-test- (testing "GET /api/database/field-values"- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}- :model/Field {f1-id :id} {:table_id t-id :name "state" :base_type :type/Text- :database_type "VARCHAR"}- :model/Field {f2-id :id} {:table_id t-id :name "rating" :base_type :type/Integer- :database_type "INTEGER"}- :model/FieldValues _ {:field_id f1-id :type :full- :values [["CA"] ["NY"] ["TX"]]- :has_more_values false}- :model/FieldValues _ {:field_id f2-id :type :full- :values [[1] [2] [3]]- :human_readable_values ["Low" "Mid" "High"]- :has_more_values true}]- (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")- by-field (into {} (map (juxt :field_id identity)) field_values)]- (is (=? {:field_id f1-id- :values [["CA"] ["NY"] ["TX"]]- :has_more_values false}- (by-field f1-id)))- (is (nil? (:human_readable_values (by-field f1-id)))- "human_readable_values is omitted when empty")- (is (=? {:field_id f2-id- :values [[1] [2] [3]]- :human_readable_values ["Low" "Mid" "High"]- :has_more_values true}- (by-field f2-id)))))))--(deftest databases-field-values-non-admin-test- (testing "GET /api/database/field-values — non-admin requests are rejected"- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}- :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text- :database_type "VARCHAR"}- :model/FieldValues _ {:field_id f-id :type :full- :values [["CA"] ["NY"]]- :has_more_values false}]- (is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :get 403 "database/field-values"))))))--(deftest databases-field-values-excludes-non-full-test- (testing "GET /api/database/field-values — only :full FieldValues are included"- (mt/with-temp [:model/Database {db-id :id} {:name "fv-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "people" :schema "PUBLIC"}- :model/Field {f-id :id} {:table_id t-id :name "state" :base_type :type/Text- :database_type "VARCHAR"}- :model/FieldValues _ {:field_id f-id :type :full- :values [["CA"]]- :has_more_values false}- :model/FieldValues _ {:field_id f-id :type :sandbox- :hash_key "sandbox-hash"- :values [["NY"]]- :has_more_values false}]- (let [{:keys [field_values]} (mt/user-http-request :crowberto :get 202 "database/field-values")- for-field (filter #(= f-id (:field_id %)) field_values)]- (is (= 1 (count for-field))- "only the :full entry streams; :sandbox / other variants are excluded")- (is (= [["CA"]] (-> for-field first :values)))))))--(deftest databases-metadata-excludes-audit-db-test- (testing "GET /api/database/metadata — audit (internal) database, its tables, and its fields are excluded"- (mt/with-temp [:model/Database {db-id :id} {:name "audit-db" :engine :h2 :is_audit true}- :model/Table {t-id :id} {:db_id db-id :name "audit_table" :schema "PUBLIC"}- :model/Field {f-id :id} {:table_id t-id :name "audit_col" :base_type :type/Integer}]- (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202 "database/metadata")]- (is (nil? (m/find-first (comp #{db-id} :id) databases)))- (is (nil? (m/find-first (comp #{t-id} :id) tables)))- (is (nil? (m/find-first (comp #{f-id} :id) fields)))))))-(deftest ^:parallel fetch-database-metadata-test(testing "GET /api/database/:id/metadata"(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)