Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

revert metadata endpoint updates (#73950)

Alexander Polyankinc71899676ab7
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index b3adb539c3e..a648babc234 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3311,7 +3311,6 @@
lib
logger
models
- permissions
plugins
premium-features
search
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 6b3e74ae9b0..4e9e129710f 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -308,13 +308,13 @@
(finally
(io/delete-file (:tempfile file)))))
-;;; ----------------------------------- GET /api/ee/serialization/export/json ------------------------------------
+;;; ----------------------------------- GET /api/ee/serialization/metadata/export ------------------------------------
-(api.macros/defendpoint :get "/export/json"
- :- (sr/streaming-response-schema ::schema/export-response)
+(api.macros/defendpoint :get "/metadata/export"
+ :- (sr/streaming-response-schema ::schema/export-metadata-response)
"Get warehouse metadata (databases, tables, and fields) for all databases visible to the
- current user, with all references emitted in serdes-portable form (database names, table
- `[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).
+ current user. References between rows are emitted as raw numeric ids (`db_id`,
+ `table_id`, `parent_id`, `fk_target_field_id`).
Sections must be opted into with the `with-databases`, `with-tables`, and `with-fields`
query parameters — they all default to `false`. The response is streamed for efficiency
@@ -330,7 +330,7 @@
(let [opts (assoc query-params :user-info {:user-id api/*current-user-id*
:is-superuser? api/*is-superuser?*})]
(sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]
- (export/export! os opts))))
+ (export/export-metadata! os opts))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/serialization` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export.clj b/enterprise/backend/src/metabase_enterprise/serialization/export.clj
index 06bf27275da..e350714a6da 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/export.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/export.clj
@@ -1,27 +1,54 @@
(ns metabase-enterprise.serialization.export
- "Orchestrates the metadata-export pipeline: walks [[sections]], runs each
- section's `export-query`, formats rows via `format-entity`, and streams the
- result as a JSON object whose keys are the section names. Sections whose
- `:enabled?` predicate returns false for `opts` are skipped."
+ "Orchestrates the metadata export pipeline: walks [[sections]], runs each
+ section's `serdes/metadata-query`, formats rows via `serdes/metadata-query-format`,
+ and streams the result as a JSON object whose keys are the section names.
+ Sections whose `:enabled?` predicate returns false for `opts` are skipped."
(:require
- [metabase-enterprise.serialization.export.format :as export.format]
- [metabase-enterprise.serialization.export.json :as export.json]
- [metabase-enterprise.serialization.export.query :as export.query]
[metabase-enterprise.serialization.schema :as schema]
+ [metabase.models.serialization :as serdes]
+ [metabase.util.json :as json]
[metabase.util.malli :as mu])
(:import
- (java.io BufferedWriter OutputStream OutputStreamWriter)
+ (java.io BufferedWriter OutputStream OutputStreamWriter Writer)
(java.nio.charset StandardCharsets)))
(set! *warn-on-reflection* true)
-(def sections
+(def ^:private sections
"Sections emitted by the metadata export, in response order."
[{:name "databases" :model :model/Database :enabled? :with-databases}
{:name "tables" :model :model/Table :enabled? :with-tables}
{:name "fields" :model :model/Field :enabled? :with-fields}])
-(mu/defn export!
+(defn- write-json-array!
+ "Streams a reducible collection as a JSON array to `writer`. `run!` is required —
+ it dispatches through `reduce`, which consumes the `IReduceInit` returned by
+ `t2/reducible-query` row-by-row without materializing."
+ [^Writer writer reducible]
+ (.write writer "[")
+ (let [first? (volatile! true)]
+ (run! (fn [row]
+ (if @first?
+ (vreset! first? false)
+ (.write writer ","))
+ (json/encode-to row writer {}))
+ reducible))
+ (.write writer "]"))
+
+(defn- write-json-object!
+ "Writes a JSON object whose values are JSON arrays. `entries` is a reducible of
+ `[entry-name reducible-rows]` pairs."
+ [^Writer writer entries]
+ (.write writer "{")
+ (let [first? (volatile! true)]
+ (run! (fn [[entry-name objects]]
+ (if @first? (vreset! first? false) (.write writer ","))
+ (.write writer (str "\"" entry-name "\":"))
+ (write-json-array! writer objects))
+ entries))
+ (.write writer "}"))
+
+(mu/defn export-metadata!
"Streams the metadata export to `os` as a JSON object. Walks [[sections]] and
includes each section only when its `:enabled?` predicate is truthy for `opts`.
@@ -29,15 +56,15 @@
pulled from reducible queries and streamed directly to the writer — memory
stays bounded regardless of schema size."
[^OutputStream os
- opts :- ::schema/export-options]
+ opts :- ::schema/export-metadata-options]
(let [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
- (export.json/write-json-object!
+ (write-json-object!
writer
(eduction
(filter (fn [{:keys [enabled?]}] (enabled? opts)))
(map (fn [{:keys [name model]}]
[name (eduction
- (map #(export.format/format-entity model %))
- (export.query/export-query model opts))]))
+ (map #(serdes/metadata-query-format model %))
+ (serdes/metadata-query model opts))]))
sections))
(.flush writer)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj
deleted file mode 100644
index 769df864de6..00000000000
--- a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj
+++ /dev/null
@@ -1,93 +0,0 @@
-(ns metabase-enterprise.serialization.export.format
- "Per-model row formatters for the metadata-export pipeline. Each row coming out
- of the export query is reshaped via `format-entity` into the JSON shape the
- endpoint emits, with all references — database, table, fk_target_field —
- rendered in serdes-portable form (names rather than numeric IDs) so the
- response can be ingested by another Metabase instance with different
- surrogate keys."
- (:require
- [medley.core :as m]
- [metabase.util.json :as json]))
-
-(set! *warn-on-reflection* true)
-
-(defn- decode-nfc-path
- "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for
- null/empty, the original value when it's already a sequential collection, or the decoded
- vector when it's a JSON string."
- [nfc-path]
- (cond
- (nil? nfc-path) nil
- (sequential? nfc-path) (seq nfc-path)
- (string? nfc-path) (seq (json/decode nfc-path))))
-
-(defn- parent-field-path
- "The nfc_path-form path of a field's parent (names from the outer-most
- ancestor down to the parent's leaf), or nil when `parent-id` is not set. For
- BigQuery the field's own `nfc_path` does not include the leaf field name and
- therefore already represents the parent path; for Mongo `nfc_path` includes
- the leaf, so the parent path is `(butlast nfc-path)`."
- [parent-id nfc-path engine]
- (when parent-id
- (seq (if (= engine "bigquery-cloud-sdk")
- nfc-path
- (butlast nfc-path)))))
-
-(defn- format-database-id
- "Portable id for a database — its name."
- [db-name]
- db-name)
-
-(defn- format-table-id
- "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."
- [db-name schema table-name]
- [db-name schema table-name])
-
-(defn- format-field-id
- "Portable id for a field — `[db schema table ...parent-path field-name]`.
- `parent-path` may be nil (or empty) for root fields, in which case the id
- collapses to `[db schema table field-name]`."
- [db-name schema table-name field-name parent-path]
- (-> [db-name schema table-name] (into parent-path) (conj field-name)))
-
-(defmulti format-entity
- "Reshapes a raw query row for `model` into the JSON shape emitted by the
- metadata-export endpoint."
- {:arglists '([model row])}
- (fn [model _row] model))
-
-(defmethod format-entity :model/Database
- [_model {:keys [name engine]}]
- {:name name :engine engine})
-
-(defmethod format-entity :model/Table
- [_model {:keys [db_name schema table_name description]}]
- (m/assoc-some {:db_id (format-database-id db_name)
- :name table_name}
- :schema schema
- :description description))
-
-(defmethod format-entity :model/Field
- [_model {:keys [db_name engine table_schema table_name field_name parent_id description
- base_type database_type effective_type semantic_type coercion_strategy nfc_path
- fk_db_name fk_db_engine fk_table_schema fk_table_name fk_field_name fk_parent_id
- fk_field_nfc_path]}]
- (let [nfc-path (decode-nfc-path nfc_path)
- parent-path (parent-field-path parent_id nfc-path engine)
- parent-id (when parent-path
- (into [db_name table_schema table_name] parent-path))
- fk-parent-path (parent-field-path fk_parent_id (decode-nfc-path fk_field_nfc_path) fk_db_engine)
- fk-target-id (when (and fk_db_name fk_table_name fk_field_name)
- (format-field-id fk_db_name fk_table_schema fk_table_name
- fk_field_name fk-parent-path))]
- (m/assoc-some {:table_id (format-table-id db_name table_schema table_name)
- :name field_name}
- :description description
- :base_type base_type
- :database_type database_type
- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
- :semantic_type semantic_type
- :coercion_strategy coercion_strategy
- :nfc_path nfc-path
- :parent_id parent-id
- :fk_target_field_id fk-target-id)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj
deleted file mode 100644
index ab3e2186ca2..00000000000
--- a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj
+++ /dev/null
@@ -1,44 +0,0 @@
-(ns metabase-enterprise.serialization.export.json
- "Streaming JSON writers used by the metadata-export pipeline. Writers consume reducible
- collections row-by-row so memory stays bounded regardless of result size."
- (:require
- [metabase.util.json :as json]))
-
-(set! *warn-on-reflection* true)
-
-(defn write-json-array!
- "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded
- directly with no transformation — apply per-row formatting via an `eduction` (or other
- transducer pipeline) before passing in.
-
- `run!` is required here because it dispatches through `reduce`, which consumes the
- `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.
- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible
- would realize every row into memory — defeating the point of streaming."
- [^java.io.Writer writer reducible]
- (.write writer "[")
- (let [first? (volatile! true)]
- (run! (fn [row]
- (if @first?
- (vreset! first? false)
- (.write writer ","))
- (json/encode-to row writer {}))
- reducible))
- (.write writer "]"))
-
-(defn write-json-object!
- "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible
- of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)
- of already-formatted values to encode.
-
- `run!` is used over the entries — `doseq` would walk a seq and realize the underlying
- reducible, defeating streaming."
- [^java.io.Writer writer entries]
- (.write writer "{")
- (let [first? (volatile! true)]
- (run! (fn [[entry-name objects]]
- (if @first? (vreset! first? false) (.write writer ","))
- (.write writer (str "\"" entry-name "\":"))
- (write-json-array! writer objects))
- entries))
- (.write writer "}"))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj
deleted file mode 100644
index 0cb829c3aac..00000000000
--- a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj
+++ /dev/null
@@ -1,111 +0,0 @@
-(ns metabase-enterprise.serialization.export.query
- "Reducible queries for the metadata-export pipeline. Each model implements
- `export-query` to produce the reducible row stream, and `export-query-filter`
- to produce a HoneySQL where predicate scoped to a given table alias. Filters
- are reusable across queries — e.g. the field query joins through table and
- database, so it composes the database/table filters too."
- (:require
- [metabase.permissions.core :as perms]
- [metabase.util :as u]
- [toucan2.core :as t2]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private perm-mapping
- "Permission mapping required to include a database/table in the export."
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder})
-
-(defmulti export-query
- "Returns a reducible row stream for `model`. `opts` carries caller context
- (e.g. `:user-info`, section flags)."
- {:arglists '([model opts])}
- (fn [model _opts] model))
-
-(defmulti export-query-filter
- "Returns a HoneySQL where predicate that restricts rows of `model` qualified by
- table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across the
- database/table/field hierarchy."
- {:arglists '([model alias opts])}
- (fn [model _alias _opts] model))
-
-(defmethod export-query-filter :model/Database
- [_model alias {:keys [user-info]}]
- [:and
- [:= (u/qualified-key alias :is_audit) false]
- [:= (u/qualified-key alias :router_database_id) nil]
- [:in (u/qualified-key alias :id) (perms/visible-database-filter-select user-info perm-mapping)]])
-
-(defmethod export-query-filter :model/Table
- [_model alias {:keys [user-info]}]
- [:and
- [:= (u/qualified-key alias :active) true]
- [:= (u/qualified-key alias :visibility_type) nil]
- [:in (u/qualified-key alias :id) (perms/visible-table-filter-select :id user-info perm-mapping)]])
-
-(defmethod export-query-filter :model/Field
- [_model alias _opts]
- [:and
- [:= (u/qualified-key alias :active) true]
- [:<> (u/qualified-key alias :visibility_type) "sensitive"]])
-
-(defmethod export-query :model/Database
- [model opts]
- (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]
- :from [[:metabase_database :db]]
- :where (export-query-filter model :db opts)}))
-
-(defmethod export-query :model/Table
- [_model opts]
- (t2/reducible-query {:select [[:db.name :db_name]
- [:table.schema :schema]
- [:table.name :table_name]
- [:table.description :description]]
- :from [[:metabase_table :table]]
- :join [[:metabase_database :db] [:= :table.db_id :db.id]]
- :where [:and
- (export-query-filter :model/Database :db opts)
- (export-query-filter :model/Table :table opts)]}))
-
-(defmethod export-query :model/Field
- [_model opts]
- (t2/reducible-query
- {:select [[:db.name :db_name]
- [:db.engine :engine]
- [:table.schema :table_schema]
- [:table.name :table_name]
- [:field.name :field_name]
- [:field.parent_id :parent_id]
- [:field.description :description]
- [:field.base_type :base_type]
- [:field.database_type :database_type]
- [:field.effective_type :effective_type]
- [:field.semantic_type :semantic_type]
- [:field.coercion_strategy :coercion_strategy]
- [:field.nfc_path :nfc_path]
- [:fk_db.name :fk_db_name]
- [:fk_db.engine :fk_db_engine]
- [:fk_table.schema :fk_table_schema]
- [:fk_table.name :fk_table_name]
- [:fk_field.name :fk_field_name]
- [:fk_field.parent_id :fk_parent_id]
- [:fk_field.nfc_path :fk_field_nfc_path]]
- :from [[:metabase_field :field]]
- :join [[:metabase_table :table] [:= :field.table_id :table.id]
- [:metabase_database :db] [:= :table.db_id :db.id]]
- :left-join [[:metabase_field :fk_field]
- [:and
- [:= :field.fk_target_field_id :fk_field.id]
- (export-query-filter :model/Field :fk_field opts)]
- [:metabase_table :fk_table]
- [:and
- [:= :fk_field.table_id :fk_table.id]
- (export-query-filter :model/Table :fk_table opts)]
- [:metabase_database :fk_db]
- [:and
- [:= :fk_table.db_id :fk_db.id]
- (export-query-filter :model/Database :fk_db opts)]]
- :where [:and
- (export-query-filter :model/Database :db opts)
- (export-query-filter :model/Table :table opts)
- (export-query-filter :model/Field :field opts)]}))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
index 4ab457ff1bb..d05dfa1c6ec 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
@@ -1,7 +1,9 @@
(ns metabase-enterprise.serialization.schema
- "Malli schemas for the serdes-portable shapes produced by
- [[metabase-enterprise.serialization.export.format]]."
+ "Malli schemas for the rows produced by the metadata export pipeline. References
+ to other entities are emitted as raw numeric ids; consumers are expected to
+ resolve them against the same Metabase instance."
(:require
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli.registry :as mr]))
(mr/def ::user-info
@@ -9,50 +11,59 @@
[:user-id pos-int?]
[:is-superuser? :boolean]])
-(mr/def ::export-options
+(mr/def ::export-metadata-options
+ "Options accepted by the metadata export pipeline:
+
+ - `:user-info` — caller identity for permission filtering.
+ - `:with-databases` — include the `databases` section.
+ - `:with-tables` — include the `tables` section.
+ - `:with-fields` — include the `fields` section.
+ - `:database-ids` — restrict databases to these ids.
+ - `:schema-ids` — `{db-id [\"schema\" ...]}` — restrict tables to these (db, schema) pairs.
+ - `:table-ids` — restrict tables to these ids.
+ - `:field-ids` — restrict fields to these ids."
[:map
[:user-info ::user-info]
[:with-databases {:optional true} [:maybe :boolean]]
[:with-tables {:optional true} [:maybe :boolean]]
- [:with-fields {:optional true} [:maybe :boolean]]])
-
-(mr/def ::external-database-id
- :string)
-
-(mr/def ::external-table-id
- [:tuple :string [:maybe :string] :string])
+ [:with-fields {:optional true} [:maybe :boolean]]
+ [:database-ids {:optional true} [:maybe [:sequential ::lib.schema.id/database]]]
+ [:schema-ids {:optional true} [:maybe [:map-of ::lib.schema.id/database
+ [:sequential :string]]]]
+ [:table-ids {:optional true} [:maybe [:sequential ::lib.schema.id/table]]]
+ [:field-ids {:optional true} [:maybe [:sequential ::lib.schema.id/field]]]])
-(mr/def ::external-field-id
- [:cat :string [:maybe :string] :string [:+ :string]])
-
-(mr/def ::external-database
+(mr/def ::exported-database
[:map
+ [:id ::lib.schema.id/database]
[:name :string]
[:engine :string]])
-(mr/def ::external-table
+(mr/def ::exported-table
[:map
- [:db_id ::external-database-id]
+ [:id ::lib.schema.id/table]
+ [:db_id ::lib.schema.id/database]
[:name :string]
[:schema {:optional true} :string]
[:description {:optional true} :string]])
-(mr/def ::external-field
+(mr/def ::exported-field
[:map
- [:table_id ::external-table-id]
+ [:id ::lib.schema.id/field]
+ [:table_id ::lib.schema.id/table]
[:name :string]
- [:parent_id {:optional true} ::external-field-id]
- [:fk_target_field_id {:optional true} ::external-field-id]
- [:description {:optional true} :string]
[:base_type :string]
+ [:description {:optional true} :string]
[:database_type {:optional true} :string]
[:effective_type {:optional true} :string]
[:semantic_type {:optional true} :string]
[:coercion_strategy {:optional true} :string]
- [:nfc_path {:optional true} [:sequential :string]]])
+ [:nfc_path {:optional true} [:sequential :string]]
+ [:parent_id {:optional true} ::lib.schema.id/field]
+ [:fk_target_field_id {:optional true} ::lib.schema.id/field]])
-(mr/def ::export-response
+(mr/def ::export-metadata-response
[:map
- [:databases {:optional true} [:sequential ::external-database]]
- [:tables {:optional true} [:sequential ::external-table]]
- [:fields {:optional true} [:sequential ::external-field]]])
+ [:databases {:optional true} [:sequential ::exported-database]]
+ [:tables {:optional true} [:sequential ::exported-table]]
+ [:fields {:optional true} [:sequential ::exported-field]]])
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 5062fef9770..12623e1b509 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -444,33 +444,33 @@
{:file (byte-array 0)})))))))
(deftest metadata-export-basic-test
- (testing "GET /api/ee/serialization/export/json — happy path with one db/table/field"
+ (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
:description "A test table"}
- :model/Field {f-name :name} {:table_id t-id
+ :model/Field {f-id :id f-name :name} {:table_id t-id
:base_type :type/Integer
:database_type "BIGINT"
:semantic_type :type/PK}]
(let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-databases true
:with-tables true
:with-fields true)
- test-db (m/find-first (comp #{db-name} :name) databases)
- test-table (m/find-first #(and (= (:db_id %) db-name) (= (:name %) t-name)) tables)
- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
- (= (:name %) f-name))
- fields)]
- (is (=? {:name db-name :engine "h2"}
+ test-db (m/find-first (comp #{db-id} :id) databases)
+ test-table (m/find-first (comp #{t-id} :id) tables)
+ test-field (m/find-first (comp #{f-id} :id) fields)]
+ (is (=? {:id db-id :name db-name :engine "h2"}
test-db))
- (is (=? {:db_id db-name
+ (is (=? {:id t-id
+ :db_id db-id
:name t-name
:schema "PUBLIC"
:description "A test table"}
test-table))
- (is (=? {:table_id [db-name "PUBLIC" t-name]
+ (is (=? {:id f-id
+ :table_id t-id
:name f-name
:base_type "type/Integer"
:database_type "BIGINT"
@@ -478,23 +478,21 @@
test-field)))))))
(deftest metadata-export-optional-properties-test
- (testing "GET /api/ee/serialization/export/json — effective_type, coercion_strategy and description are emitted when set"
+ (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {f-name :name} {:table_id t-id
- :base_type :type/Text
- :database_type "VARCHAR"
- :effective_type :type/DateTime
- :coercion_strategy :Coercion/ISO8601->DateTime
- :description "When this happened"}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {f-id :id} {:table_id t-id
+ :base_type :type/Text
+ :database_type "VARCHAR"
+ :effective_type :type/DateTime
+ :coercion_strategy :Coercion/ISO8601->DateTime
+ :description "When this happened"}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-fields true)
- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
- (= (:name %) f-name))
- fields)]
- (is (=? {:name f-name
+ test-field (m/find-first (comp #{f-id} :id) fields)]
+ (is (=? {:id f-id
:base_type "type/Text"
:effective_type "type/DateTime"
:coercion_strategy "Coercion/ISO8601->DateTime"
@@ -502,195 +500,150 @@
test-field)))))))
(deftest metadata-export-parent-field-test
- (testing "GET /api/ee/serialization/export/json — parent_id is emitted only when raw field.parent_id is set"
+ (testing "GET /api/ee/serialization/metadata/export — parent_id is emitted only when raw field.parent_id is set; nfc_path is decoded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {root-name :name} {:table_id t-id
- :base_type :type/Text}
- :model/Field {parent-id :id} {:table_id t-id
- :name "data"
- :base_type :type/Dictionary}
- :model/Field _ {:table_id t-id
- :name "city"
- :base_type :type/Text
- :parent_id parent-id
- :nfc_path ["data" "city"]}]
- (let [table-id [db-name "PUBLIC" t-name]
- {:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {root-id :id} {:table_id t-id
+ :base_type :type/Text}
+ :model/Field {parent-id :id} {:table_id t-id
+ :name "data"
+ :base_type :type/Dictionary}
+ :model/Field {child-id :id} {:table_id t-id
+ :name "city"
+ :base_type :type/Text
+ :parent_id parent-id
+ :nfc_path ["data" "city"]}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
:with-fields true)
- test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) root-name)) fields)
- test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) "city")) fields)]
- (is (=? {:name root-name :table_id table-id} test-field-1))
- (is (not (contains? test-field-1 :parent_id)))
- (is (=? {:name "city"
- :table_id table-id
- :parent_id [db-name "PUBLIC" t-name "data"]}
- test-field-2)))))))
+ root-field (m/find-first (comp #{root-id} :id) fields)
+ child-field (m/find-first (comp #{child-id} :id) fields)]
+ (is (=? {:id root-id :table_id t-id} root-field))
+ (is (not (contains? root-field :parent_id)))
+ (is (=? {:id child-id
+ :table_id t-id
+ :parent_id parent-id
+ :nfc_path ["data" "city"]}
+ child-field)))))))
(deftest metadata-export-fk-test
- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is emitted in portable form"
+ (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted as a raw id"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {pk-id :id pk-name :name} {:table_id t-id
- :base_type :type/Integer
- :semantic_type :type/PK}
- :model/Field {fk-name :name} {:table_id t-id
- :base_type :type/Integer
- :semantic_type :type/FK
- :fk_target_field_id pk-id}]
- (let [table-id [db-name "PUBLIC" t-name]
- {:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {pk-id :id} {:table_id t-id
+ :base_type :type/Integer
+ :semantic_type :type/PK}
+ :model/Field {fk-id :id} {:table_id t-id
+ :base_type :type/Integer
+ :semantic_type :type/FK
+ :fk_target_field_id pk-id}]
+ (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/metadata/export"
:with-fields true)
- test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) pk-name)) fields)
- test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) fk-name)) fields)]
- (is (=? {:name pk-name :table_id table-id :semantic_type "type/PK"}
- test-field-1))
- (is (=? {:name fk-name
- :table_id table-id
+ pk-field (m/find-first (comp #{pk-id} :id) fields)
+ fk-field (m/find-first (comp #{fk-id} :id) fields)]
+ (is (=? {:id pk-id :table_id t-id :semantic_type "type/PK"}
+ pk-field))
+ (is (=? {:id fk-id
+ :table_id t-id
:semantic_type "type/FK"
- :fk_target_field_id [db-name "PUBLIC" t-name pk-name]}
- test-field-2)))))))
+ :fk_target_field_id pk-id}
+ fk-field)))))))
(deftest metadata-export-hidden-table-test
- (testing "GET /api/ee/serialization/export/json — hidden tables and their fields are excluded"
+ (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
- :model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Table {hidden-id :id hidden-t-name :name} {:db_id db-id :schema "PUBLIC"
- :visibility_type :hidden}
- :model/Field {visible-f-name :name} {:table_id visible-id
- :base_type :type/Integer}
- :model/Field {hidden-f-name :name} {:table_id hidden-id
- :base_type :type/Integer}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {visible-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"
+ :visibility_type :hidden}
+ :model/Field {visible-f-id :id} {:table_id visible-id
+ :base_type :type/Integer}
+ :model/Field {hidden-f-id :id} {:table_id hidden-id
+ :base_type :type/Integer}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-tables true :with-fields true)]
- (is (some (comp #{visible-t-name} :name) tables))
- (is (not-any? (comp #{hidden-t-name} :name) tables))
- (is (some (comp #{visible-f-name} :name) fields))
- (is (not-any? (comp #{hidden-f-name} :name) fields)))))))
-
-(deftest metadata-export-hidden-table-fk-test
- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is hidden"
- (mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"
- :visibility_type :hidden}
- :model/Field {target-id :id} {:table_id hidden-id
- :base_type :type/Integer
- :semantic_type :type/PK}
- :model/Field {fk-name :name} {:table_id visible-id
- :base_type :type/Integer
- :semantic_type :type/FK
- :fk_target_field_id target-id}]
- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
- :with-fields true)
- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" v-name])
- (= (:name %) fk-name))
- fields)]
- (is (some? test-field))
- (is (not (contains? test-field :fk_target_field_id))))))))
+ (is (some (comp #{visible-id} :id) tables))
+ (is (not-any? (comp #{hidden-id} :id) tables))
+ (is (some (comp #{visible-f-id} :id) fields))
+ (is (not-any? (comp #{hidden-f-id} :id) fields)))))))
(deftest metadata-export-inactive-table-test
- (testing "GET /api/ee/serialization/export/json — inactive tables and all of their fields are excluded"
+ (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
- :active false}
- :model/Field {active-f-name :name} {:table_id t-id
- :base_type :type/Integer}
- :model/Field {inactive-f-name :name} {:table_id t-id
- :base_type :type/Integer
- :active false}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"
+ :active false}
+ :model/Field {active-f-id :id} {:table_id t-id
+ :base_type :type/Integer}
+ :model/Field {inactive-f-id :id} {:table_id t-id
+ :base_type :type/Integer
+ :active false}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-tables true :with-fields true)]
- (is (not-any? (comp #{t-name} :name) tables))
- (is (not-any? (comp #{active-f-name} :name) fields))
- (is (not-any? (comp #{inactive-f-name} :name) fields)))))))
+ (is (not-any? (comp #{t-id} :id) tables))
+ (is (not-any? (comp #{active-f-id} :id) fields))
+ (is (not-any? (comp #{inactive-f-id} :id) fields)))))))
(deftest metadata-export-sensitive-field-test
- (testing "GET /api/ee/serialization/export/json — sensitive fields are excluded"
+ (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
- :model/Field {email-name :name} {:table_id t-id :base_type :type/Text}
- :model/Field {secret-name :name} {:table_id t-id :base_type :type/Text
- :visibility_type :sensitive}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {email-id :id} {:table_id t-id :base_type :type/Text}
+ :model/Field {secret-id :id} {:table_id t-id :base_type :type/Text
+ :visibility_type :sensitive}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-fields true)]
- (is (some (comp #{email-name} :name) fields))
- (is (not-any? (comp #{secret-name} :name) fields)))))))
-
-(deftest metadata-export-sensitive-fk-test
- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target field is sensitive"
- (mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {target-id :id} {:table_id t-id
- :base_type :type/Text
- :visibility_type :sensitive}
- :model/Field {fk-name :name} {:table_id t-id
- :base_type :type/Text
- :semantic_type :type/FK
- :fk_target_field_id target-id}]
- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
- :with-fields true)
- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
- (= (:name %) fk-name))
- fields)]
- (is (some? test-field))
- (is (not (contains? test-field :fk_target_field_id))))))))
+ (is (some (comp #{email-id} :id) fields))
+ (is (not-any? (comp #{secret-id} :id) fields)))))))
(deftest metadata-export-inactive-field-test
- (testing "GET /api/ee/serialization/export/json — inactive fields are excluded; their active sibling and table still appear"
+ (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {active-f-name :name} {:table_id t-id
- :base_type :type/Integer}
- :model/Field {inactive-f-name :name} {:table_id t-id
- :base_type :type/Integer
- :active false}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {active-f-id :id} {:table_id t-id
+ :base_type :type/Integer}
+ :model/Field {inactive-f-id :id} {:table_id t-id
+ :base_type :type/Integer
+ :active false}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-tables true :with-fields true)]
- (is (some (comp #{t-name} :name) tables))
- (is (some (comp #{active-f-name} :name) fields))
- (is (not-any? (comp #{inactive-f-name} :name) fields)))))))
+ (is (some (comp #{t-id} :id) tables))
+ (is (some (comp #{active-f-id} :id) fields))
+ (is (not-any? (comp #{inactive-f-id} :id) fields)))))))
(deftest metadata-export-db-routing-test
- (testing "GET /api/ee/serialization/export/json — router (mirror) databases are excluded"
+ (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database _ {:engine :h2}
- :model/Database {primary-id :id primary-name :name} {:engine :h2}
- :model/Database {mirror-name :name} {:engine :h2
- :router_database_id primary-id}]
+ (mt/with-temp [:model/Database _ {:engine :h2}
+ :model/Database {primary-id :id} {:engine :h2}
+ :model/Database {mirror-id :id} {:engine :h2
+ :router_database_id primary-id}]
(let [{:keys [databases]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-databases true)]
- (is (some (comp #{primary-name} :name) databases))
- (is (not-any? (comp #{mirror-name} :name) databases)))))))
+ (is (some (comp #{primary-id} :id) databases))
+ (is (not-any? (comp #{mirror-id} :id) databases)))))))
(deftest metadata-export-table-permission-test
- (testing "GET /api/ee/serialization/export/json — tables the current user can't access are excluded"
+ (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"
(mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
- :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Table {restricted-id :id restricted-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Field {ok-f-name :name} {:table_id accessible-id
- :base_type :type/Integer}
- :model/Field {no-f-name :name} {:table_id restricted-id
- :base_type :type/Integer}
- :model/PermissionsGroup pg {}]
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {accessible-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}
+ :model/Field {ok-f-id :id} {:table_id accessible-id
+ :base_type :type/Integer}
+ :model/Field {no-f-id :id} {:table_id restricted-id
+ :base_type :type/Integer}
+ :model/PermissionsGroup pg {}]
(perms/add-user-to-group! (mt/user->id :rasta) pg)
(t2/delete! :model/DataPermissions :db_id db-id)
(perms/set-database-permission! pg db-id :perms/view-data :blocked)
@@ -698,48 +651,19 @@
(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
(let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202
- "ee/serialization/export/json"
+ "ee/serialization/metadata/export"
:with-tables true :with-fields true)]
- (is (some (comp #{accessible-name} :name) tables))
- (is (not-any? (comp #{restricted-name} :name) tables))
- (is (some (comp #{ok-f-name} :name) fields))
- (is (not-any? (comp #{no-f-name} :name) fields)))))))
-
-(deftest metadata-export-table-permission-fk-test
- (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is not accessible"
- (mt/with-premium-features #{:serialization}
- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
- :model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
- :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}
- :model/Field {target-id :id} {:table_id restricted-id
- :base_type :type/Integer
- :semantic_type :type/PK}
- :model/Field {fk-name :name} {:table_id accessible-id
- :base_type :type/Integer
- :semantic_type :type/FK
- :fk_target_field_id target-id}
- :model/PermissionsGroup pg {}]
- (perms/add-user-to-group! (mt/user->id :rasta) pg)
- (t2/delete! :model/DataPermissions :db_id db-id)
- (perms/set-database-permission! pg db-id :perms/view-data :blocked)
- (perms/set-database-permission! pg db-id :perms/create-queries :no)
- (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
- (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
- (let [{:keys [fields]} (mt/user-http-request :rasta :get 202
- "ee/serialization/export/json"
- :with-fields true)
- test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" accessible-name])
- (= (:name %) fk-name))
- fields)]
- (is (some? test-field))
- (is (not (contains? test-field :fk_target_field_id))))))))
+ (is (some (comp #{accessible-id} :id) tables))
+ (is (not-any? (comp #{restricted-id} :id) tables))
+ (is (some (comp #{ok-f-id} :id) fields))
+ (is (not-any? (comp #{no-f-id} :id) fields)))))))
(deftest metadata-export-token-feature-test
- (testing "GET /api/ee/serialization/export/json requires the :serialization premium feature"
+ (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"
(mt/with-premium-features #{}
(mt/assert-has-premium-feature-error
"Serialization"
- (mt/user-http-request :crowberto :get 402 "ee/serialization/export/json")))))
+ (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))
(deftest serialization-cleanup-test
(testing "No temp files are left behind after export/import operations"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj
deleted file mode 100644
index df4ae776485..00000000000
--- a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj
+++ /dev/null
@@ -1,156 +0,0 @@
-(ns metabase-enterprise.serialization.export.format-test
- "Unit tests for `format-entity` in
- `metabase-enterprise.serialization.export.format`. Each deftest exercises a
- single case so that failures pinpoint the exact behavior that broke."
- (:require
- [clojure.test :refer [deftest is]]
- [metabase-enterprise.serialization.export.format :as export.format]))
-
-(defn- mock-database
- "Default :model/Database query row, with per-test overrides merged in."
- [overrides]
- (merge {:name "Sample" :engine "h2"} overrides))
-
-(defn- mock-table
- "Default :model/Table query row, with per-test overrides merged in."
- [overrides]
- (merge {:db_name "Sample"
- :schema "PUBLIC"
- :table_name "Orders"
- :description nil}
- overrides))
-
-(defn- mock-field
- "Default :model/Field query row, with per-test overrides merged in. Covers
- every key destructured by `format-entity :model/Field`, so tests only need to
- set the fields relevant to what they're checking."
- [overrides]
- (merge {:db_name "Sample"
- :engine "h2"
- :table_schema "PUBLIC"
- :table_name "Orders"
- :field_name "id"
- :parent_id nil
- :description nil
- :base_type "type/Integer"
- :database_type nil
- :effective_type nil
- :semantic_type nil
- :coercion_strategy nil
- :nfc_path nil
- :fk_db_name nil
- :fk_db_engine nil
- :fk_table_schema nil
- :fk_table_name nil
- :fk_field_name nil
- :fk_parent_id nil
- :fk_field_nfc_path nil}
- overrides))
-
-(deftest format-database-test
- (is (=? {:name "Sample" :engine "h2"}
- (export.format/format-entity :model/Database (mock-database {})))))
-
-(deftest format-table-with-schema-and-description-test
- (is (=? {:db_id "Sample"
- :name "Orders"
- :schema "PUBLIC"
- :description "Orders table"}
- (export.format/format-entity :model/Table
- (mock-table {:description "Orders table"})))))
-
-(deftest format-table-without-schema-test
- (let [out (export.format/format-entity :model/Table (mock-table {:schema nil}))]
- (is (=? {:db_id "Sample" :name "Orders"} out))
- (is (not (contains? out :schema)))
- (is (not (contains? out :description)))))
-
-(deftest format-field-root-test
- (let [out (export.format/format-entity :model/Field (mock-field {}))]
- (is (=? {:table_id ["Sample" "PUBLIC" "Orders"]
- :name "id"
- :base_type "type/Integer"}
- out))
- (is (not (contains? out :parent_id)))
- (is (not (contains? out :fk_target_field_id)))))
-
-(deftest format-field-mongo-nested-parent-id-test
- (is (=? {:name "city"
- :parent_id ["Sample" "PUBLIC" "Orders" "data"]}
- (export.format/format-entity :model/Field
- (mock-field {:engine "mongo"
- :field_name "city"
- :parent_id 42
- :base_type "type/Text"
- :nfc_path ["data" "city"]})))))
-
-(deftest format-field-bigquery-nested-parent-id-test
- (is (=? {:name "city"
- :parent_id ["Sample" nil "orders" "data"]}
- (export.format/format-entity :model/Field
- (mock-field {:engine "bigquery-cloud-sdk"
- :table_schema nil
- :table_name "orders"
- :field_name "city"
- :parent_id 42
- :base_type "type/Text"
- :nfc_path ["data"]})))))
-
-(deftest format-field-no-parent-id-when-raw-parent-id-missing-test
- (let [out (export.format/format-entity :model/Field
- (mock-field {:engine "mongo"
- :field_name "city"
- :base_type "type/Text"
- :nfc_path ["data" "city"]}))]
- (is (not (contains? out :parent_id)))))
-
-(deftest format-field-fk-target-root-test
- (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "id"]}
- (export.format/format-entity :model/Field
- (mock-field {:field_name "user_id"
- :fk_db_name "Sample"
- :fk_db_engine "h2"
- :fk_table_schema "PUBLIC"
- :fk_table_name "Orders"
- :fk_field_name "id"})))))
-
-(deftest format-field-fk-target-mongo-nested-test
- (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "data" "city"]}
- (export.format/format-entity :model/Field
- (mock-field {:engine "mongo"
- :table_name "Customers"
- :field_name "city_ref"
- :base_type "type/Text"
- :fk_db_name "Sample"
- :fk_db_engine "mongo"
- :fk_table_schema "PUBLIC"
- :fk_table_name "Orders"
- :fk_field_name "city"
- :fk_parent_id 99
- :fk_field_nfc_path ["data" "city"]})))))
-
-(deftest format-field-fk-target-bigquery-nested-test
- (is (=? {:fk_target_field_id ["Sample" nil "orders" "data" "city"]}
- (export.format/format-entity :model/Field
- (mock-field {:table_name "Customers"
- :field_name "city_ref"
- :base_type "type/Text"
- :fk_db_name "Sample"
- :fk_db_engine "bigquery-cloud-sdk"
- :fk_table_schema nil
- :fk_table_name "orders"
- :fk_field_name "city"
- :fk_parent_id 99
- :fk_field_nfc_path ["data"]})))))
-
-(deftest format-field-effective-type-redundant-with-base-type-test
- (let [out (export.format/format-entity :model/Field
- (mock-field {:effective_type "type/Integer"}))]
- (is (not (contains? out :effective_type)))))
-
-(deftest format-field-effective-type-distinct-from-base-type-test
- (is (=? {:effective_type "type/Date"}
- (export.format/format-entity :model/Field
- (mock-field {:field_name "created_at"
- :base_type "type/Text"
- :effective_type "type/Date"})))))
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 21afea472fb..654a7959e22 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -630,6 +630,36 @@
(defmethod required :default [_ _]
nil)
+;;; # Metadata Export
+
+(defmulti metadata-query
+ "Returns a reducible row stream for `model` for the metadata export endpoint.
+ `opts` carries caller context (e.g. `:user-info`, section flags, id allow-lists).
+ Implementations return rows with raw numeric ids — joins for portable identifiers
+ are intentionally avoided so the queries stay simple."
+ {:arglists '([model opts])}
+ (fn [model _opts] model))
+
+(defmulti metadata-query-filter
+ "Returns a HoneySQL where predicate that restricts rows of `model` qualified by
+ table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across model
+ hierarchies (e.g. the field query reuses the database/table filters)."
+ {:arglists '([model alias opts])}
+ (fn [model _alias _opts] model))
+
+(defmulti metadata-query-format
+ "Reshapes a raw query row produced by [[metadata-query]] into the JSON shape emitted
+ by the metadata export endpoint. The default drops nil-valued keys via
+ [[metabase.util/remove-nils]]; overrides do JSON decoding or column-derived
+ adjustments and rely on the same nil-pruning to keep optional keys out of the
+ response."
+ {:arglists '([model row])}
+ (fn [model _row] model))
+
+(defmethod metadata-query-format :default
+ [_model row]
+ (u/remove-nils row))
+
;;; # Import Process
;;; Deserialization is split into two stages, mirroring serialization. They are called *ingestion* and *loading*.
;;; Ingestion turns whatever serialized form was produced by storage (eg. a tree of YAML files) into Clojure maps with
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 834ae746098..ef7c55193e6 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -15,6 +15,7 @@
[metabase.remote-sync.core :as remote-sync]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -479,3 +480,42 @@
(conj (serdes/storage-path-prefixes path)
{:label "fields"}
{:label field-name :key field-name})))
+
+(defmethod serdes/metadata-query :model/Field
+ [model opts]
+ (t2/reducible-query
+ {:select [[:f.id :id]
+ [:f.table_id :table_id]
+ [:f.name :name]
+ [:f.parent_id :parent_id]
+ [:f.fk_target_field_id :fk_target_field_id]
+ [:f.description :description]
+ [:f.base_type :base_type]
+ [:f.database_type :database_type]
+ [:f.effective_type :effective_type]
+ [:f.semantic_type :semantic_type]
+ [:f.coercion_strategy :coercion_strategy]
+ [:f.nfc_path :nfc_path]]
+ :from [[(t2/table-name model) :f]]
+ :join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]
+ [(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]
+ :where [:and
+ (serdes/metadata-query-filter :model/Database :db opts)
+ (serdes/metadata-query-filter :model/Table :t opts)
+ (serdes/metadata-query-filter model :f opts)]}))
+
+(defmethod serdes/metadata-query-filter :model/Field
+ [_model alias {:keys [field-ids]}]
+ (cond-> [:and
+ [:= (u/qualified-key alias :active) true]
+ [:<> (u/qualified-key alias :visibility_type) "sensitive"]]
+ (seq field-ids) (conj [:in (u/qualified-key alias :id) field-ids])))
+
+(defmethod serdes/metadata-query-format :model/Field
+ [_model {:keys [base_type effective_type nfc_path] :as row}]
+ (-> row
+ (assoc :effective_type (when (not= base_type effective_type) effective_type))
+ (assoc :nfc_path (cond-> nfc_path
+ (string? nfc_path) json/decode
+ nfc_path seq))
+ u/remove-nils))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 1abdbb08607..069b0731bf3 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -615,6 +615,36 @@
(conj (serdes/storage-path-prefixes (serdes/path table))
{:label (:name table) :key (:name table)}))
+(defmethod serdes/metadata-query :model/Table
+ [model opts]
+ (t2/reducible-query
+ {:select [[:t.id :id]
+ [:t.db_id :db_id]
+ [:t.name :name]
+ [:t.schema :schema]
+ [:t.description :description]]
+ :from [[(t2/table-name model) :t]]
+ :join [[(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]
+ :where [:and
+ (serdes/metadata-query-filter :model/Database :db opts)
+ (serdes/metadata-query-filter model :t opts)]}))
+
+(defmethod serdes/metadata-query-filter :model/Table
+ [_model alias {:keys [user-info table-ids schema-ids]}]
+ (let [perm-mapping {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}]
+ (cond-> [:and
+ [:= (u/qualified-key alias :active) true]
+ [:= (u/qualified-key alias :visibility_type) nil]
+ [:in (u/qualified-key alias :id)
+ (perms/visible-table-filter-select :id user-info perm-mapping)]]
+ (seq schema-ids) (conj (into [:or]
+ (for [[db-id schemas] schema-ids]
+ [:and
+ [:= (u/qualified-key alias :db_id) db-id]
+ [:in (u/qualified-key alias :schema) schemas]])))
+ (seq table-ids) (conj [:in (u/qualified-key alias :id) table-ids]))))
+
;;;; ------------------------------------------------- Search ----------------------------------------------------------
(search.spec/define-spec "table"
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 17462d880df..b4b6a113cd8 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -668,6 +668,25 @@
(:write_data_details ingested) (update :write_data_details driver/sanitize-db-details))
maybe-local))
+(def ^:private metadata-export-perms
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})
+
+(defmethod serdes/metadata-query :model/Database
+ [model opts]
+ (t2/reducible-query {:select [:id :name :engine]
+ :from [[(t2/table-name model) :db]]
+ :where (serdes/metadata-query-filter model :db opts)}))
+
+(defmethod serdes/metadata-query-filter :model/Database
+ [_model alias {:keys [user-info database-ids]}]
+ (cond-> [:and
+ [:= (u/qualified-key alias :is_audit) false]
+ [:= (u/qualified-key alias :router_database_id) nil]
+ [:in (u/qualified-key alias :id)
+ (perms/visible-database-filter-select user-info metadata-export-perms)]]
+ (seq database-ids) (conj [:in (u/qualified-key alias :id) database-ids])))
+
(def ^{:arglists '([table-id])} table-id->database-id
"Retrieve the `Database` ID for the given table-id."
(mdb/memoize-for-application-db