The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
revert metadata endpoint updates (#73950)
Alexander Polyankinc71899676ab7
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex b3adb539c3e..a648babc234 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3311,7 +3311,6 @@libloggermodels- permissionspluginspremium-featuressearchdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 6b3e74ae9b0..4e9e129710f 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -308,13 +308,13 @@(finally(io/delete-file (:tempfile file)))))-;;; ----------------------------------- GET /api/ee/serialization/export/json ------------------------------------+;;; ----------------------------------- GET /api/ee/serialization/metadata/export -------------------------------------(api.macros/defendpoint :get "/export/json"- :- (sr/streaming-response-schema ::schema/export-response)+(api.macros/defendpoint :get "/metadata/export"+ :- (sr/streaming-response-schema ::schema/export-metadata-response)"Get warehouse metadata (databases, tables, and fields) for all databases visible to the- current user, with all references emitted in serdes-portable form (database names, table- `[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).+ current user. References between rows are emitted as raw numeric ids (`db_id`,+ `table_id`, `parent_id`, `fk_target_field_id`).Sections must be opted into with the `with-databases`, `with-tables`, and `with-fields`query parameters — they all default to `false`. The response is streamed for efficiency@@ -330,7 +330,7 @@(let [opts (assoc query-params :user-info {:user-id api/*current-user-id*:is-superuser? api/*is-superuser?*})](sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]- (export/export! os opts))))+ (export/export-metadata! os opts))))(def ^{:arglists '([request respond raise])} routes"`/api/ee/serialization` routes."diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export.clj b/enterprise/backend/src/metabase_enterprise/serialization/export.cljindex 06bf27275da..e350714a6da 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/export.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/export.clj@@ -1,27 +1,54 @@(ns metabase-enterprise.serialization.export- "Orchestrates the metadata-export pipeline: walks [[sections]], runs each- section's `export-query`, formats rows via `format-entity`, and streams the- result as a JSON object whose keys are the section names. Sections whose- `:enabled?` predicate returns false for `opts` are skipped."+ "Orchestrates the metadata export pipeline: walks [[sections]], runs each+ section's `serdes/metadata-query`, formats rows via `serdes/metadata-query-format`,+ and streams the result as a JSON object whose keys are the section names.+ Sections whose `:enabled?` predicate returns false for `opts` are skipped."(:require- [metabase-enterprise.serialization.export.format :as export.format]- [metabase-enterprise.serialization.export.json :as export.json]- [metabase-enterprise.serialization.export.query :as export.query][metabase-enterprise.serialization.schema :as schema]+ [metabase.models.serialization :as serdes]+ [metabase.util.json :as json][metabase.util.malli :as mu])(:import- (java.io BufferedWriter OutputStream OutputStreamWriter)+ (java.io BufferedWriter OutputStream OutputStreamWriter Writer)(java.nio.charset StandardCharsets)))(set! *warn-on-reflection* true)-(def sections+(def ^:private sections"Sections emitted by the metadata export, in response order."[{:name "databases" :model :model/Database :enabled? :with-databases}{:name "tables" :model :model/Table :enabled? :with-tables}{:name "fields" :model :model/Field :enabled? :with-fields}])-(mu/defn export!+(defn- write-json-array!+ "Streams a reducible collection as a JSON array to `writer`. `run!` is required —+ it dispatches through `reduce`, which consumes the `IReduceInit` returned by+ `t2/reducible-query` row-by-row without materializing."+ [^Writer writer reducible]+ (.write writer "[")+ (let [first? (volatile! true)]+ (run! (fn [row]+ (if @first?+ (vreset! first? false)+ (.write writer ","))+ (json/encode-to row writer {}))+ reducible))+ (.write writer "]"))++(defn- write-json-object!+ "Writes a JSON object whose values are JSON arrays. `entries` is a reducible of+ `[entry-name reducible-rows]` pairs."+ [^Writer writer entries]+ (.write writer "{")+ (let [first? (volatile! true)]+ (run! (fn [[entry-name objects]]+ (if @first? (vreset! first? false) (.write writer ","))+ (.write writer (str "\"" entry-name "\":"))+ (write-json-array! writer objects))+ entries))+ (.write writer "}"))++(mu/defn export-metadata!"Streams the metadata export to `os` as a JSON object. Walks [[sections]] andincludes each section only when its `:enabled?` predicate is truthy for `opts`.@@ -29,15 +56,15 @@pulled from reducible queries and streamed directly to the writer — memorystays bounded regardless of schema size."[^OutputStream os- opts :- ::schema/export-options]+ opts :- ::schema/export-metadata-options](let [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]- (export.json/write-json-object!+ (write-json-object!writer(eduction(filter (fn [{:keys [enabled?]}] (enabled? opts)))(map (fn [{:keys [name model]}][name (eduction- (map #(export.format/format-entity model %))- (export.query/export-query model opts))]))+ (map #(serdes/metadata-query-format model %))+ (serdes/metadata-query model opts))]))sections))(.flush writer)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/format.cljdeleted file mode 100644index 769df864de6..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj+++ /dev/null@@ -1,93 +0,0 @@-(ns metabase-enterprise.serialization.export.format- "Per-model row formatters for the metadata-export pipeline. Each row coming out- of the export query is reshaped via `format-entity` into the JSON shape the- endpoint emits, with all references — database, table, fk_target_field —- rendered in serdes-portable form (names rather than numeric IDs) so the- response can be ingested by another Metabase instance with different- surrogate keys."- (:require- [medley.core :as m]- [metabase.util.json :as json]))--(set! *warn-on-reflection* true)--(defn- decode-nfc-path- "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for- null/empty, the original value when it's already a sequential collection, or the decoded- vector when it's a JSON string."- [nfc-path]- (cond- (nil? nfc-path) nil- (sequential? nfc-path) (seq nfc-path)- (string? nfc-path) (seq (json/decode nfc-path))))--(defn- parent-field-path- "The nfc_path-form path of a field's parent (names from the outer-most- ancestor down to the parent's leaf), or nil when `parent-id` is not set. For- BigQuery the field's own `nfc_path` does not include the leaf field name and- therefore already represents the parent path; for Mongo `nfc_path` includes- the leaf, so the parent path is `(butlast nfc-path)`."- [parent-id nfc-path engine]- (when parent-id- (seq (if (= engine "bigquery-cloud-sdk")- nfc-path- (butlast nfc-path)))))--(defn- format-database-id- "Portable id for a database — its name."- [db-name]- db-name)--(defn- format-table-id- "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."- [db-name schema table-name]- [db-name schema table-name])--(defn- format-field-id- "Portable id for a field — `[db schema table ...parent-path field-name]`.- `parent-path` may be nil (or empty) for root fields, in which case the id- collapses to `[db schema table field-name]`."- [db-name schema table-name field-name parent-path]- (-> [db-name schema table-name] (into parent-path) (conj field-name)))--(defmulti format-entity- "Reshapes a raw query row for `model` into the JSON shape emitted by the- metadata-export endpoint."- {:arglists '([model row])}- (fn [model _row] model))--(defmethod format-entity :model/Database- [_model {:keys [name engine]}]- {:name name :engine engine})--(defmethod format-entity :model/Table- [_model {:keys [db_name schema table_name description]}]- (m/assoc-some {:db_id (format-database-id db_name)- :name table_name}- :schema schema- :description description))--(defmethod format-entity :model/Field- [_model {:keys [db_name engine table_schema table_name field_name parent_id description- base_type database_type effective_type semantic_type coercion_strategy nfc_path- fk_db_name fk_db_engine fk_table_schema fk_table_name fk_field_name fk_parent_id- fk_field_nfc_path]}]- (let [nfc-path (decode-nfc-path nfc_path)- parent-path (parent-field-path parent_id nfc-path engine)- parent-id (when parent-path- (into [db_name table_schema table_name] parent-path))- fk-parent-path (parent-field-path fk_parent_id (decode-nfc-path fk_field_nfc_path) fk_db_engine)- fk-target-id (when (and fk_db_name fk_table_name fk_field_name)- (format-field-id fk_db_name fk_table_schema fk_table_name- fk_field_name fk-parent-path))]- (m/assoc-some {:table_id (format-table-id db_name table_schema table_name)- :name field_name}- :description description- :base_type base_type- :database_type database_type- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)- :semantic_type semantic_type- :coercion_strategy coercion_strategy- :nfc_path nfc-path- :parent_id parent-id- :fk_target_field_id fk-target-id)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/json.cljdeleted file mode 100644index ab3e2186ca2..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj+++ /dev/null@@ -1,44 +0,0 @@-(ns metabase-enterprise.serialization.export.json- "Streaming JSON writers used by the metadata-export pipeline. Writers consume reducible- collections row-by-row so memory stays bounded regardless of result size."- (:require- [metabase.util.json :as json]))--(set! *warn-on-reflection* true)--(defn write-json-array!- "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded- directly with no transformation — apply per-row formatting via an `eduction` (or other- transducer pipeline) before passing in.-- `run!` is required here because it dispatches through `reduce`, which consumes the- `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible- would realize every row into memory — defeating the point of streaming."- [^java.io.Writer writer reducible]- (.write writer "[")- (let [first? (volatile! true)]- (run! (fn [row]- (if @first?- (vreset! first? false)- (.write writer ","))- (json/encode-to row writer {}))- reducible))- (.write writer "]"))--(defn write-json-object!- "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible- of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)- of already-formatted values to encode.-- `run!` is used over the entries — `doseq` would walk a seq and realize the underlying- reducible, defeating streaming."- [^java.io.Writer writer entries]- (.write writer "{")- (let [first? (volatile! true)]- (run! (fn [[entry-name objects]]- (if @first? (vreset! first? false) (.write writer ","))- (.write writer (str "\"" entry-name "\":"))- (write-json-array! writer objects))- entries))- (.write writer "}"))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/query.cljdeleted file mode 100644index 0cb829c3aac..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj+++ /dev/null@@ -1,111 +0,0 @@-(ns metabase-enterprise.serialization.export.query- "Reducible queries for the metadata-export pipeline. Each model implements- `export-query` to produce the reducible row stream, and `export-query-filter`- to produce a HoneySQL where predicate scoped to a given table alias. Filters- are reusable across queries — e.g. the field query joins through table and- database, so it composes the database/table filters too."- (:require- [metabase.permissions.core :as perms]- [metabase.util :as u]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(def ^:private perm-mapping- "Permission mapping required to include a database/table in the export."- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})--(defmulti export-query- "Returns a reducible row stream for `model`. `opts` carries caller context- (e.g. `:user-info`, section flags)."- {:arglists '([model opts])}- (fn [model _opts] model))--(defmulti export-query-filter- "Returns a HoneySQL where predicate that restricts rows of `model` qualified by- table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across the- database/table/field hierarchy."- {:arglists '([model alias opts])}- (fn [model _alias _opts] model))--(defmethod export-query-filter :model/Database- [_model alias {:keys [user-info]}]- [:and- [:= (u/qualified-key alias :is_audit) false]- [:= (u/qualified-key alias :router_database_id) nil]- [:in (u/qualified-key alias :id) (perms/visible-database-filter-select user-info perm-mapping)]])--(defmethod export-query-filter :model/Table- [_model alias {:keys [user-info]}]- [:and- [:= (u/qualified-key alias :active) true]- [:= (u/qualified-key alias :visibility_type) nil]- [:in (u/qualified-key alias :id) (perms/visible-table-filter-select :id user-info perm-mapping)]])--(defmethod export-query-filter :model/Field- [_model alias _opts]- [:and- [:= (u/qualified-key alias :active) true]- [:<> (u/qualified-key alias :visibility_type) "sensitive"]])--(defmethod export-query :model/Database- [model opts]- (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]- :from [[:metabase_database :db]]- :where (export-query-filter model :db opts)}))--(defmethod export-query :model/Table- [_model opts]- (t2/reducible-query {:select [[:db.name :db_name]- [:table.schema :schema]- [:table.name :table_name]- [:table.description :description]]- :from [[:metabase_table :table]]- :join [[:metabase_database :db] [:= :table.db_id :db.id]]- :where [:and- (export-query-filter :model/Database :db opts)- (export-query-filter :model/Table :table opts)]}))--(defmethod export-query :model/Field- [_model opts]- (t2/reducible-query- {:select [[:db.name :db_name]- [:db.engine :engine]- [:table.schema :table_schema]- [:table.name :table_name]- [:field.name :field_name]- [:field.parent_id :parent_id]- [:field.description :description]- [:field.base_type :base_type]- [:field.database_type :database_type]- [:field.effective_type :effective_type]- [:field.semantic_type :semantic_type]- [:field.coercion_strategy :coercion_strategy]- [:field.nfc_path :nfc_path]- [:fk_db.name :fk_db_name]- [:fk_db.engine :fk_db_engine]- [:fk_table.schema :fk_table_schema]- [:fk_table.name :fk_table_name]- [:fk_field.name :fk_field_name]- [:fk_field.parent_id :fk_parent_id]- [:fk_field.nfc_path :fk_field_nfc_path]]- :from [[:metabase_field :field]]- :join [[:metabase_table :table] [:= :field.table_id :table.id]- [:metabase_database :db] [:= :table.db_id :db.id]]- :left-join [[:metabase_field :fk_field]- [:and- [:= :field.fk_target_field_id :fk_field.id]- (export-query-filter :model/Field :fk_field opts)]- [:metabase_table :fk_table]- [:and- [:= :fk_field.table_id :fk_table.id]- (export-query-filter :model/Table :fk_table opts)]- [:metabase_database :fk_db]- [:and- [:= :fk_table.db_id :fk_db.id]- (export-query-filter :model/Database :fk_db opts)]]- :where [:and- (export-query-filter :model/Database :db opts)- (export-query-filter :model/Table :table opts)- (export-query-filter :model/Field :field opts)]}))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.cljindex 4ab457ff1bb..d05dfa1c6ec 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.serialization.schema- "Malli schemas for the serdes-portable shapes produced by- [[metabase-enterprise.serialization.export.format]]."+ "Malli schemas for the rows produced by the metadata export pipeline. References+ to other entities are emitted as raw numeric ids; consumers are expected to+ resolve them against the same Metabase instance."(:require+ [metabase.lib.schema.id :as lib.schema.id][metabase.util.malli.registry :as mr]))(mr/def ::user-info@@ -9,50 +11,59 @@[:user-id pos-int?][:is-superuser? :boolean]])-(mr/def ::export-options+(mr/def ::export-metadata-options+ "Options accepted by the metadata export pipeline:++ - `:user-info` — caller identity for permission filtering.+ - `:with-databases` — include the `databases` section.+ - `:with-tables` — include the `tables` section.+ - `:with-fields` — include the `fields` section.+ - `:database-ids` — restrict databases to these ids.+ - `:schema-ids` — `{db-id [\"schema\" ...]}` — restrict tables to these (db, schema) pairs.+ - `:table-ids` — restrict tables to these ids.+ - `:field-ids` — restrict fields to these ids."[:map[:user-info ::user-info][:with-databases {:optional true} [:maybe :boolean]][:with-tables {:optional true} [:maybe :boolean]]- [:with-fields {:optional true} [:maybe :boolean]]])--(mr/def ::external-database-id- :string)--(mr/def ::external-table-id- [:tuple :string [:maybe :string] :string])+ [:with-fields {:optional true} [:maybe :boolean]]+ [:database-ids {:optional true} [:maybe [:sequential ::lib.schema.id/database]]]+ [:schema-ids {:optional true} [:maybe [:map-of ::lib.schema.id/database+ [:sequential :string]]]]+ [:table-ids {:optional true} [:maybe [:sequential ::lib.schema.id/table]]]+ [:field-ids {:optional true} [:maybe [:sequential ::lib.schema.id/field]]]])-(mr/def ::external-field-id- [:cat :string [:maybe :string] :string [:+ :string]])--(mr/def ::external-database+(mr/def ::exported-database[:map+ [:id ::lib.schema.id/database][:name :string][:engine :string]])-(mr/def ::external-table+(mr/def ::exported-table[:map- [:db_id ::external-database-id]+ [:id ::lib.schema.id/table]+ [:db_id ::lib.schema.id/database][:name :string][:schema {:optional true} :string][:description {:optional true} :string]])-(mr/def ::external-field+(mr/def ::exported-field[:map- [:table_id ::external-table-id]+ [:id ::lib.schema.id/field]+ [:table_id ::lib.schema.id/table][:name :string]- [:parent_id {:optional true} ::external-field-id]- [:fk_target_field_id {:optional true} ::external-field-id]- [:description {:optional true} :string][:base_type :string]+ [:description {:optional true} :string][:database_type {:optional true} :string][:effective_type {:optional true} :string][:semantic_type {:optional true} :string][:coercion_strategy {:optional true} :string]- [:nfc_path {:optional true} [:sequential :string]]])+ [:nfc_path {:optional true} [:sequential :string]]+ [:parent_id {:optional true} ::lib.schema.id/field]+ [:fk_target_field_id {:optional true} ::lib.schema.id/field]])-(mr/def ::export-response+(mr/def ::export-metadata-response[:map- [:databases {:optional true} [:sequential ::external-database]]- [:tables {:optional true} [:sequential ::external-table]]- [:fields {:optional true} [:sequential ::external-field]]])+ [:databases {:optional true} [:sequential ::exported-database]]+ [:tables {:optional true} [:sequential ::exported-table]]+ [:fields {:optional true} [:sequential ::exported-field]]])diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 5062fef9770..12623e1b509 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -444,33 +444,33 @@{:file (byte-array 0)})))))))(deftest metadata-export-basic-test- (testing "GET /api/ee/serialization/export/json — happy path with one db/table/field"+ (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC":description "A test table"}- :model/Field {f-name :name} {:table_id t-id+ :model/Field {f-id :id f-name :name} {:table_id t-id:base_type :type/Integer:database_type "BIGINT":semantic_type :type/PK}](let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-databases true:with-tables true:with-fields true)- test-db (m/find-first (comp #{db-name} :name) databases)- test-table (m/find-first #(and (= (:db_id %) db-name) (= (:name %) t-name)) tables)- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])- (= (:name %) f-name))- fields)]- (is (=? {:name db-name :engine "h2"}+ test-db (m/find-first (comp #{db-id} :id) databases)+ test-table (m/find-first (comp #{t-id} :id) tables)+ test-field (m/find-first (comp #{f-id} :id) fields)]+ (is (=? {:id db-id :name db-name :engine "h2"}test-db))- (is (=? {:db_id db-name+ (is (=? {:id t-id+ :db_id db-id:name t-name:schema "PUBLIC":description "A test table"}test-table))- (is (=? {:table_id [db-name "PUBLIC" t-name]+ (is (=? {:id f-id+ :table_id t-id:name f-name:base_type "type/Integer":database_type "BIGINT"@@ -478,23 +478,21 @@test-field)))))))(deftest metadata-export-optional-properties-test- (testing "GET /api/ee/serialization/export/json — effective_type, coercion_strategy and description are emitted when set"+ (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {f-name :name} {:table_id t-id- :base_type :type/Text- :database_type "VARCHAR"- :effective_type :type/DateTime- :coercion_strategy :Coercion/ISO8601->DateTime- :description "When this happened"}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {f-id :id} {:table_id t-id+ :base_type :type/Text+ :database_type "VARCHAR"+ :effective_type :type/DateTime+ :coercion_strategy :Coercion/ISO8601->DateTime+ :description "When this happened"}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-fields true)- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])- (= (:name %) f-name))- fields)]- (is (=? {:name f-name+ test-field (m/find-first (comp #{f-id} :id) fields)]+ (is (=? {:id f-id:base_type "type/Text":effective_type "type/DateTime":coercion_strategy "Coercion/ISO8601->DateTime"@@ -502,195 +500,150 @@test-field)))))))(deftest metadata-export-parent-field-test- (testing "GET /api/ee/serialization/export/json — parent_id is emitted only when raw field.parent_id is set"+ (testing "GET /api/ee/serialization/metadata/export — parent_id is emitted only when raw field.parent_id is set; nfc_path is decoded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {root-name :name} {:table_id t-id- :base_type :type/Text}- :model/Field {parent-id :id} {:table_id t-id- :name "data"- :base_type :type/Dictionary}- :model/Field _ {:table_id t-id- :name "city"- :base_type :type/Text- :parent_id parent-id- :nfc_path ["data" "city"]}]- (let [table-id [db-name "PUBLIC" t-name]- {:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {root-id :id} {:table_id t-id+ :base_type :type/Text}+ :model/Field {parent-id :id} {:table_id t-id+ :name "data"+ :base_type :type/Dictionary}+ :model/Field {child-id :id} {:table_id t-id+ :name "city"+ :base_type :type/Text+ :parent_id parent-id+ :nfc_path ["data" "city"]}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export":with-fields true)- test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) root-name)) fields)- test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) "city")) fields)]- (is (=? {:name root-name :table_id table-id} test-field-1))- (is (not (contains? test-field-1 :parent_id)))- (is (=? {:name "city"- :table_id table-id- :parent_id [db-name "PUBLIC" t-name "data"]}- test-field-2)))))))+ root-field (m/find-first (comp #{root-id} :id) fields)+ child-field (m/find-first (comp #{child-id} :id) fields)]+ (is (=? {:id root-id :table_id t-id} root-field))+ (is (not (contains? root-field :parent_id)))+ (is (=? {:id child-id+ :table_id t-id+ :parent_id parent-id+ :nfc_path ["data" "city"]}+ child-field)))))))(deftest metadata-export-fk-test- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is emitted in portable form"+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted as a raw id"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {pk-id :id pk-name :name} {:table_id t-id- :base_type :type/Integer- :semantic_type :type/PK}- :model/Field {fk-name :name} {:table_id t-id- :base_type :type/Integer- :semantic_type :type/FK- :fk_target_field_id pk-id}]- (let [table-id [db-name "PUBLIC" t-name]- {:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {pk-id :id} {:table_id t-id+ :base_type :type/Integer+ :semantic_type :type/PK}+ :model/Field {fk-id :id} {:table_id t-id+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id pk-id}]+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/metadata/export":with-fields true)- test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) pk-name)) fields)- test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) fk-name)) fields)]- (is (=? {:name pk-name :table_id table-id :semantic_type "type/PK"}- test-field-1))- (is (=? {:name fk-name- :table_id table-id+ pk-field (m/find-first (comp #{pk-id} :id) fields)+ fk-field (m/find-first (comp #{fk-id} :id) fields)]+ (is (=? {:id pk-id :table_id t-id :semantic_type "type/PK"}+ pk-field))+ (is (=? {:id fk-id+ :table_id t-id:semantic_type "type/FK"- :fk_target_field_id [db-name "PUBLIC" t-name pk-name]}- test-field-2)))))))+ :fk_target_field_id pk-id}+ fk-field)))))))(deftest metadata-export-hidden-table-test- (testing "GET /api/ee/serialization/export/json — hidden tables and their fields are excluded"+ (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Table {hidden-id :id hidden-t-name :name} {:db_id db-id :schema "PUBLIC"- :visibility_type :hidden}- :model/Field {visible-f-name :name} {:table_id visible-id- :base_type :type/Integer}- :model/Field {hidden-f-name :name} {:table_id hidden-id- :base_type :type/Integer}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {visible-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"+ :visibility_type :hidden}+ :model/Field {visible-f-id :id} {:table_id visible-id+ :base_type :type/Integer}+ :model/Field {hidden-f-id :id} {:table_id hidden-id+ :base_type :type/Integer}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-tables true :with-fields true)]- (is (some (comp #{visible-t-name} :name) tables))- (is (not-any? (comp #{hidden-t-name} :name) tables))- (is (some (comp #{visible-f-name} :name) fields))- (is (not-any? (comp #{hidden-f-name} :name) fields)))))))--(deftest metadata-export-hidden-table-fk-test- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is hidden"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"- :visibility_type :hidden}- :model/Field {target-id :id} {:table_id hidden-id- :base_type :type/Integer- :semantic_type :type/PK}- :model/Field {fk-name :name} {:table_id visible-id- :base_type :type/Integer- :semantic_type :type/FK- :fk_target_field_id target-id}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"- :with-fields true)- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" v-name])- (= (:name %) fk-name))- fields)]- (is (some? test-field))- (is (not (contains? test-field :fk_target_field_id))))))))+ (is (some (comp #{visible-id} :id) tables))+ (is (not-any? (comp #{hidden-id} :id) tables))+ (is (some (comp #{visible-f-id} :id) fields))+ (is (not-any? (comp #{hidden-f-id} :id) fields)))))))(deftest metadata-export-inactive-table-test- (testing "GET /api/ee/serialization/export/json — inactive tables and all of their fields are excluded"+ (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"- :active false}- :model/Field {active-f-name :name} {:table_id t-id- :base_type :type/Integer}- :model/Field {inactive-f-name :name} {:table_id t-id- :base_type :type/Integer- :active false}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"+ :active false}+ :model/Field {active-f-id :id} {:table_id t-id+ :base_type :type/Integer}+ :model/Field {inactive-f-id :id} {:table_id t-id+ :base_type :type/Integer+ :active false}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-tables true :with-fields true)]- (is (not-any? (comp #{t-name} :name) tables))- (is (not-any? (comp #{active-f-name} :name) fields))- (is (not-any? (comp #{inactive-f-name} :name) fields)))))))+ (is (not-any? (comp #{t-id} :id) tables))+ (is (not-any? (comp #{active-f-id} :id) fields))+ (is (not-any? (comp #{inactive-f-id} :id) fields)))))))(deftest metadata-export-sensitive-field-test- (testing "GET /api/ee/serialization/export/json — sensitive fields are excluded"+ (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {email-name :name} {:table_id t-id :base_type :type/Text}- :model/Field {secret-name :name} {:table_id t-id :base_type :type/Text- :visibility_type :sensitive}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {email-id :id} {:table_id t-id :base_type :type/Text}+ :model/Field {secret-id :id} {:table_id t-id :base_type :type/Text+ :visibility_type :sensitive}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-fields true)]- (is (some (comp #{email-name} :name) fields))- (is (not-any? (comp #{secret-name} :name) fields)))))))--(deftest metadata-export-sensitive-fk-test- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target field is sensitive"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {target-id :id} {:table_id t-id- :base_type :type/Text- :visibility_type :sensitive}- :model/Field {fk-name :name} {:table_id t-id- :base_type :type/Text- :semantic_type :type/FK- :fk_target_field_id target-id}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"- :with-fields true)- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])- (= (:name %) fk-name))- fields)]- (is (some? test-field))- (is (not (contains? test-field :fk_target_field_id))))))))+ (is (some (comp #{email-id} :id) fields))+ (is (not-any? (comp #{secret-id} :id) fields)))))))(deftest metadata-export-inactive-field-test- (testing "GET /api/ee/serialization/export/json — inactive fields are excluded; their active sibling and table still appear"+ (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {active-f-name :name} {:table_id t-id- :base_type :type/Integer}- :model/Field {inactive-f-name :name} {:table_id t-id- :base_type :type/Integer- :active false}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {active-f-id :id} {:table_id t-id+ :base_type :type/Integer}+ :model/Field {inactive-f-id :id} {:table_id t-id+ :base_type :type/Integer+ :active false}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-tables true :with-fields true)]- (is (some (comp #{t-name} :name) tables))- (is (some (comp #{active-f-name} :name) fields))- (is (not-any? (comp #{inactive-f-name} :name) fields)))))))+ (is (some (comp #{t-id} :id) tables))+ (is (some (comp #{active-f-id} :id) fields))+ (is (not-any? (comp #{inactive-f-id} :id) fields)))))))(deftest metadata-export-db-routing-test- (testing "GET /api/ee/serialization/export/json — router (mirror) databases are excluded"+ (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database _ {:engine :h2}- :model/Database {primary-id :id primary-name :name} {:engine :h2}- :model/Database {mirror-name :name} {:engine :h2- :router_database_id primary-id}]+ (mt/with-temp [:model/Database _ {:engine :h2}+ :model/Database {primary-id :id} {:engine :h2}+ :model/Database {mirror-id :id} {:engine :h2+ :router_database_id primary-id}](let [{:keys [databases]} (mt/user-http-request :crowberto :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-databases true)]- (is (some (comp #{primary-name} :name) databases))- (is (not-any? (comp #{mirror-name} :name) databases)))))))+ (is (some (comp #{primary-id} :id) databases))+ (is (not-any? (comp #{mirror-id} :id) databases)))))))(deftest metadata-export-table-permission-test- (testing "GET /api/ee/serialization/export/json — tables the current user can't access are excluded"+ (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"(mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Table {restricted-id :id restricted-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Field {ok-f-name :name} {:table_id accessible-id- :base_type :type/Integer}- :model/Field {no-f-name :name} {:table_id restricted-id- :base_type :type/Integer}- :model/PermissionsGroup pg {}]+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {accessible-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}+ :model/Field {ok-f-id :id} {:table_id accessible-id+ :base_type :type/Integer}+ :model/Field {no-f-id :id} {:table_id restricted-id+ :base_type :type/Integer}+ :model/PermissionsGroup pg {}](perms/add-user-to-group! (mt/user->id :rasta) pg)(t2/delete! :model/DataPermissions :db_id db-id)(perms/set-database-permission! pg db-id :perms/view-data :blocked)@@ -698,48 +651,19 @@(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)(let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202- "ee/serialization/export/json"+ "ee/serialization/metadata/export":with-tables true :with-fields true)]- (is (some (comp #{accessible-name} :name) tables))- (is (not-any? (comp #{restricted-name} :name) tables))- (is (some (comp #{ok-f-name} :name) fields))- (is (not-any? (comp #{no-f-name} :name) fields)))))))--(deftest metadata-export-table-permission-fk-test- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is not accessible"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}- :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {target-id :id} {:table_id restricted-id- :base_type :type/Integer- :semantic_type :type/PK}- :model/Field {fk-name :name} {:table_id accessible-id- :base_type :type/Integer- :semantic_type :type/FK- :fk_target_field_id target-id}- :model/PermissionsGroup pg {}]- (perms/add-user-to-group! (mt/user->id :rasta) pg)- (t2/delete! :model/DataPermissions :db_id db-id)- (perms/set-database-permission! pg db-id :perms/view-data :blocked)- (perms/set-database-permission! pg db-id :perms/create-queries :no)- (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)- (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)- (let [{:keys [fields]} (mt/user-http-request :rasta :get 202- "ee/serialization/export/json"- :with-fields true)- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" accessible-name])- (= (:name %) fk-name))- fields)]- (is (some? test-field))- (is (not (contains? test-field :fk_target_field_id))))))))+ (is (some (comp #{accessible-id} :id) tables))+ (is (not-any? (comp #{restricted-id} :id) tables))+ (is (some (comp #{ok-f-id} :id) fields))+ (is (not-any? (comp #{no-f-id} :id) fields)))))))(deftest metadata-export-token-feature-test- (testing "GET /api/ee/serialization/export/json requires the :serialization premium feature"+ (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"(mt/with-premium-features #{}(mt/assert-has-premium-feature-error"Serialization"- (mt/user-http-request :crowberto :get 402 "ee/serialization/export/json")))))+ (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))(deftest serialization-cleanup-test(testing "No temp files are left behind after export/import operations"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.cljdeleted file mode 100644index df4ae776485..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj+++ /dev/null@@ -1,156 +0,0 @@-(ns metabase-enterprise.serialization.export.format-test- "Unit tests for `format-entity` in- `metabase-enterprise.serialization.export.format`. Each deftest exercises a- single case so that failures pinpoint the exact behavior that broke."- (:require- [clojure.test :refer [deftest is]]- [metabase-enterprise.serialization.export.format :as export.format]))--(defn- mock-database- "Default :model/Database query row, with per-test overrides merged in."- [overrides]- (merge {:name "Sample" :engine "h2"} overrides))--(defn- mock-table- "Default :model/Table query row, with per-test overrides merged in."- [overrides]- (merge {:db_name "Sample"- :schema "PUBLIC"- :table_name "Orders"- :description nil}- overrides))--(defn- mock-field- "Default :model/Field query row, with per-test overrides merged in. Covers- every key destructured by `format-entity :model/Field`, so tests only need to- set the fields relevant to what they're checking."- [overrides]- (merge {:db_name "Sample"- :engine "h2"- :table_schema "PUBLIC"- :table_name "Orders"- :field_name "id"- :parent_id nil- :description nil- :base_type "type/Integer"- :database_type nil- :effective_type nil- :semantic_type nil- :coercion_strategy nil- :nfc_path nil- :fk_db_name nil- :fk_db_engine nil- :fk_table_schema nil- :fk_table_name nil- :fk_field_name nil- :fk_parent_id nil- :fk_field_nfc_path nil}- overrides))--(deftest format-database-test- (is (=? {:name "Sample" :engine "h2"}- (export.format/format-entity :model/Database (mock-database {})))))--(deftest format-table-with-schema-and-description-test- (is (=? {:db_id "Sample"- :name "Orders"- :schema "PUBLIC"- :description "Orders table"}- (export.format/format-entity :model/Table- (mock-table {:description "Orders table"})))))--(deftest format-table-without-schema-test- (let [out (export.format/format-entity :model/Table (mock-table {:schema nil}))]- (is (=? {:db_id "Sample" :name "Orders"} out))- (is (not (contains? out :schema)))- (is (not (contains? out :description)))))--(deftest format-field-root-test- (let [out (export.format/format-entity :model/Field (mock-field {}))]- (is (=? {:table_id ["Sample" "PUBLIC" "Orders"]- :name "id"- :base_type "type/Integer"}- out))- (is (not (contains? out :parent_id)))- (is (not (contains? out :fk_target_field_id)))))--(deftest format-field-mongo-nested-parent-id-test- (is (=? {:name "city"- :parent_id ["Sample" "PUBLIC" "Orders" "data"]}- (export.format/format-entity :model/Field- (mock-field {:engine "mongo"- :field_name "city"- :parent_id 42- :base_type "type/Text"- :nfc_path ["data" "city"]})))))--(deftest format-field-bigquery-nested-parent-id-test- (is (=? {:name "city"- :parent_id ["Sample" nil "orders" "data"]}- (export.format/format-entity :model/Field- (mock-field {:engine "bigquery-cloud-sdk"- :table_schema nil- :table_name "orders"- :field_name "city"- :parent_id 42- :base_type "type/Text"- :nfc_path ["data"]})))))--(deftest format-field-no-parent-id-when-raw-parent-id-missing-test- (let [out (export.format/format-entity :model/Field- (mock-field {:engine "mongo"- :field_name "city"- :base_type "type/Text"- :nfc_path ["data" "city"]}))]- (is (not (contains? out :parent_id)))))--(deftest format-field-fk-target-root-test- (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "id"]}- (export.format/format-entity :model/Field- (mock-field {:field_name "user_id"- :fk_db_name "Sample"- :fk_db_engine "h2"- :fk_table_schema "PUBLIC"- :fk_table_name "Orders"- :fk_field_name "id"})))))--(deftest format-field-fk-target-mongo-nested-test- (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "data" "city"]}- (export.format/format-entity :model/Field- (mock-field {:engine "mongo"- :table_name "Customers"- :field_name "city_ref"- :base_type "type/Text"- :fk_db_name "Sample"- :fk_db_engine "mongo"- :fk_table_schema "PUBLIC"- :fk_table_name "Orders"- :fk_field_name "city"- :fk_parent_id 99- :fk_field_nfc_path ["data" "city"]})))))--(deftest format-field-fk-target-bigquery-nested-test- (is (=? {:fk_target_field_id ["Sample" nil "orders" "data" "city"]}- (export.format/format-entity :model/Field- (mock-field {:table_name "Customers"- :field_name "city_ref"- :base_type "type/Text"- :fk_db_name "Sample"- :fk_db_engine "bigquery-cloud-sdk"- :fk_table_schema nil- :fk_table_name "orders"- :fk_field_name "city"- :fk_parent_id 99- :fk_field_nfc_path ["data"]})))))--(deftest format-field-effective-type-redundant-with-base-type-test- (let [out (export.format/format-entity :model/Field- (mock-field {:effective_type "type/Integer"}))]- (is (not (contains? out :effective_type)))))--(deftest format-field-effective-type-distinct-from-base-type-test- (is (=? {:effective_type "type/Date"}- (export.format/format-entity :model/Field- (mock-field {:field_name "created_at"- :base_type "type/Text"- :effective_type "type/Date"})))))diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 21afea472fb..654a7959e22 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -630,6 +630,36 @@(defmethod required :default [_ _]nil)+;;; # Metadata Export++(defmulti metadata-query+ "Returns a reducible row stream for `model` for the metadata export endpoint.+ `opts` carries caller context (e.g. `:user-info`, section flags, id allow-lists).+ Implementations return rows with raw numeric ids — joins for portable identifiers+ are intentionally avoided so the queries stay simple."+ {:arglists '([model opts])}+ (fn [model _opts] model))++(defmulti metadata-query-filter+ "Returns a HoneySQL where predicate that restricts rows of `model` qualified by+ table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across model+ hierarchies (e.g. the field query reuses the database/table filters)."+ {:arglists '([model alias opts])}+ (fn [model _alias _opts] model))++(defmulti metadata-query-format+ "Reshapes a raw query row produced by [[metadata-query]] into the JSON shape emitted+ by the metadata export endpoint. The default drops nil-valued keys via+ [[metabase.util/remove-nils]]; overrides do JSON decoding or column-derived+ adjustments and rely on the same nil-pruning to keep optional keys out of the+ response."+ {:arglists '([model row])}+ (fn [model _row] model))++(defmethod metadata-query-format :default+ [_model row]+ (u/remove-nils row))+;;; # Import Process;;; Deserialization is split into two stages, mirroring serialization. They are called *ingestion* and *loading*.;;; Ingestion turns whatever serialized form was produced by storage (eg. a tree of YAML files) into Clojure maps withdiff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 834ae746098..ef7c55193e6 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -15,6 +15,7 @@[metabase.remote-sync.core :as remote-sync][metabase.util :as u][metabase.util.i18n :refer [tru]]+ [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -479,3 +480,42 @@(conj (serdes/storage-path-prefixes path){:label "fields"}{:label field-name :key field-name})))++(defmethod serdes/metadata-query :model/Field+ [model opts]+ (t2/reducible-query+ {:select [[:f.id :id]+ [:f.table_id :table_id]+ [:f.name :name]+ [:f.parent_id :parent_id]+ [:f.fk_target_field_id :fk_target_field_id]+ [:f.description :description]+ [:f.base_type :base_type]+ [:f.database_type :database_type]+ [:f.effective_type :effective_type]+ [:f.semantic_type :semantic_type]+ [:f.coercion_strategy :coercion_strategy]+ [:f.nfc_path :nfc_path]]+ :from [[(t2/table-name model) :f]]+ :join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]+ [(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]+ :where [:and+ (serdes/metadata-query-filter :model/Database :db opts)+ (serdes/metadata-query-filter :model/Table :t opts)+ (serdes/metadata-query-filter model :f opts)]}))++(defmethod serdes/metadata-query-filter :model/Field+ [_model alias {:keys [field-ids]}]+ (cond-> [:and+ [:= (u/qualified-key alias :active) true]+ [:<> (u/qualified-key alias :visibility_type) "sensitive"]]+ (seq field-ids) (conj [:in (u/qualified-key alias :id) field-ids])))++(defmethod serdes/metadata-query-format :model/Field+ [_model {:keys [base_type effective_type nfc_path] :as row}]+ (-> row+ (assoc :effective_type (when (not= base_type effective_type) effective_type))+ (assoc :nfc_path (cond-> nfc_path+ (string? nfc_path) json/decode+ nfc_path seq))+ u/remove-nils))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 1abdbb08607..069b0731bf3 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -615,6 +615,36 @@(conj (serdes/storage-path-prefixes (serdes/path table)){:label (:name table) :key (:name table)}))+(defmethod serdes/metadata-query :model/Table+ [model opts]+ (t2/reducible-query+ {:select [[:t.id :id]+ [:t.db_id :db_id]+ [:t.name :name]+ [:t.schema :schema]+ [:t.description :description]]+ :from [[(t2/table-name model) :t]]+ :join [[(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]+ :where [:and+ (serdes/metadata-query-filter :model/Database :db opts)+ (serdes/metadata-query-filter model :t opts)]}))++(defmethod serdes/metadata-query-filter :model/Table+ [_model alias {:keys [user-info table-ids schema-ids]}]+ (let [perm-mapping {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}]+ (cond-> [:and+ [:= (u/qualified-key alias :active) true]+ [:= (u/qualified-key alias :visibility_type) nil]+ [:in (u/qualified-key alias :id)+ (perms/visible-table-filter-select :id user-info perm-mapping)]]+ (seq schema-ids) (conj (into [:or]+ (for [[db-id schemas] schema-ids]+ [:and+ [:= (u/qualified-key alias :db_id) db-id]+ [:in (u/qualified-key alias :schema) schemas]])))+ (seq table-ids) (conj [:in (u/qualified-key alias :id) table-ids]))))+;;;; ------------------------------------------------- Search ----------------------------------------------------------(search.spec/define-spec "table"diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 17462d880df..b4b6a113cd8 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -668,6 +668,25 @@(:write_data_details ingested) (update :write_data_details driver/sanitize-db-details))maybe-local))+(def ^:private metadata-export-perms+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})++(defmethod serdes/metadata-query :model/Database+ [model opts]+ (t2/reducible-query {:select [:id :name :engine]+ :from [[(t2/table-name model) :db]]+ :where (serdes/metadata-query-filter model :db opts)}))++(defmethod serdes/metadata-query-filter :model/Database+ [_model alias {:keys [user-info database-ids]}]+ (cond-> [:and+ [:= (u/qualified-key alias :is_audit) false]+ [:= (u/qualified-key alias :router_database_id) nil]+ [:in (u/qualified-key alias :id)+ (perms/visible-database-filter-select user-info metadata-export-perms)]]+ (seq database-ids) (conj [:in (u/qualified-key alias :id) database-ids])))+(def ^{:arglists '([table-id])} table-id->database-id"Retrieve the `Database` ID for the given table-id."(mdb/memoize-for-application-db