Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add clear per-model slots for json-based serialization (#73851)

Alexander Polyankin83547976da39
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 32b3e195a77..6b3e74ae9b0 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -3,7 +3,7 @@
[clojure.java.io :as io]
[clojure.string :as str]
[java-time.api :as t]
- [metabase-enterprise.serialization.metadata :as metadata]
+ [metabase-enterprise.serialization.export :as export]
[metabase-enterprise.serialization.schema :as schema]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
@@ -22,7 +22,6 @@
[metabase.util :as u]
[metabase.util.compress :as u.compress]
[metabase.util.date-2 :as u.date]
- [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -309,67 +308,10 @@
(finally
(io/delete-file (:tempfile file)))))
-;;; ----------------------------------- GET /api/ee/serialization/metadata/export -----------------------------------
+;;; ----------------------------------- GET /api/ee/serialization/export/json ------------------------------------
-(defn- write-json-array!
- "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded
- directly with no transformation — apply per-row formatting via an `eduction` (or other
- transducer pipeline) before passing in.
-
- `run!` is required here because it dispatches through `reduce`, which consumes the
- `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.
- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible
- would realize every row into memory — defeating the point of streaming."
- [^java.io.Writer writer reducible]
- (.write writer "[")
- (let [first? (volatile! true)]
- (run! (fn [row]
- (if @first?
- (vreset! first? false)
- (.write writer ","))
- (json/encode-to row writer {}))
- reducible))
- (.write writer "]"))
-
-(defn- write-json-object!
- "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible
- of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)
- of already-formatted values to encode.
-
- `run!` is used over the entries — `doseq` would walk a seq and realize the underlying
- reducible, defeating streaming."
- [^java.io.Writer writer entries]
- (.write writer "{")
- (let [first? (volatile! true)]
- (run! (fn [[entry-name objects]]
- (if @first? (vreset! first? false) (.write writer ","))
- (.write writer (str "\"" entry-name "\":"))
- (write-json-array! writer objects))
- entries))
- (.write writer "}"))
-
-(defn- write-databases-metadata!
- "Streams the databases/tables/fields metadata to the given OutputStream. Sections are
- included only when their `with-...?` flag is true. All references — database, table,
- fk_target_field — are emitted in serdes-portable form (names rather than numeric IDs)
- so the response can be ingested by another Metabase instance with different surrogate
- keys.
-
- Warehouses with large schemas can produce gigabytes of metadata, so rows are pulled
- from reducible queries and streamed directly to the writer — memory stays bounded
- regardless of schema size."
- [^java.io.OutputStream os {:keys [with-databases? with-tables? with-fields?]}]
- (let [writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]
- (write-json-object!
- writer
- (cond-> []
- with-databases? (conj ["databases" (metadata/reducible-databases)])
- with-tables? (conj ["tables" (metadata/reducible-tables)])
- with-fields? (conj ["fields" (metadata/reducible-fields)])))
- (.flush writer)))
-
-(api.macros/defendpoint :get "/metadata/export"
- :- (sr/streaming-response-schema ::schema/metadata-export-response)
+(api.macros/defendpoint :get "/export/json"
+ :- (sr/streaming-response-schema ::schema/export-response)
"Get warehouse metadata (databases, tables, and fields) for all databases visible to the
current user, with all references emitted in serdes-portable form (database names, table
`[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).
@@ -381,16 +323,14 @@
Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or
`Query builder and native`) permissions on each database and table."
[_route-params
- {:keys [with-databases with-tables with-fields]}
- :- [:map
- [:with-databases {:default false} [:maybe :boolean]]
- [:with-tables {:default false} [:maybe :boolean]]
- [:with-fields {:default false} [:maybe :boolean]]]]
- (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]
- (write-databases-metadata! os
- {:with-databases? with-databases
- :with-tables? with-tables
- :with-fields? with-fields})))
+ query-params :- [:map
+ [:with-databases {:default false} [:maybe :boolean]]
+ [:with-tables {:default false} [:maybe :boolean]]
+ [:with-fields {:default false} [:maybe :boolean]]]]
+ (let [opts (assoc query-params :user-info {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*})]
+ (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]
+ (export/export! os opts))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/serialization` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export.clj b/enterprise/backend/src/metabase_enterprise/serialization/export.clj
new file mode 100644
index 00000000000..06bf27275da
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/export.clj
@@ -0,0 +1,43 @@
+(ns metabase-enterprise.serialization.export
+ "Orchestrates the metadata-export pipeline: walks [[sections]], runs each
+ section's `export-query`, formats rows via `format-entity`, and streams the
+ result as a JSON object whose keys are the section names. Sections whose
+ `:enabled?` predicate returns false for `opts` are skipped."
+ (:require
+ [metabase-enterprise.serialization.export.format :as export.format]
+ [metabase-enterprise.serialization.export.json :as export.json]
+ [metabase-enterprise.serialization.export.query :as export.query]
+ [metabase-enterprise.serialization.schema :as schema]
+ [metabase.util.malli :as mu])
+ (:import
+ (java.io BufferedWriter OutputStream OutputStreamWriter)
+ (java.nio.charset StandardCharsets)))
+
+(set! *warn-on-reflection* true)
+
+(def sections
+ "Sections emitted by the metadata export, in response order."
+ [{:name "databases" :model :model/Database :enabled? :with-databases}
+ {:name "tables" :model :model/Table :enabled? :with-tables}
+ {:name "fields" :model :model/Field :enabled? :with-fields}])
+
+(mu/defn export!
+ "Streams the metadata export to `os` as a JSON object. Walks [[sections]] and
+ includes each section only when its `:enabled?` predicate is truthy for `opts`.
+
+ Warehouses with large schemas can produce gigabytes of metadata, so rows are
+ pulled from reducible queries and streamed directly to the writer — memory
+ stays bounded regardless of schema size."
+ [^OutputStream os
+ opts :- ::schema/export-options]
+ (let [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
+ (export.json/write-json-object!
+ writer
+ (eduction
+ (filter (fn [{:keys [enabled?]}] (enabled? opts)))
+ (map (fn [{:keys [name model]}]
+ [name (eduction
+ (map #(export.format/format-entity model %))
+ (export.query/export-query model opts))]))
+ sections))
+ (.flush writer)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj
new file mode 100644
index 00000000000..769df864de6
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj
@@ -0,0 +1,93 @@
+(ns metabase-enterprise.serialization.export.format
+ "Per-model row formatters for the metadata-export pipeline. Each row coming out
+ of the export query is reshaped via `format-entity` into the JSON shape the
+ endpoint emits, with all references — database, table, fk_target_field —
+ rendered in serdes-portable form (names rather than numeric IDs) so the
+ response can be ingested by another Metabase instance with different
+ surrogate keys."
+ (:require
+ [medley.core :as m]
+ [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
+
+(defn- decode-nfc-path
+ "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for
+ null/empty, the original value when it's already a sequential collection, or the decoded
+ vector when it's a JSON string."
+ [nfc-path]
+ (cond
+ (nil? nfc-path) nil
+ (sequential? nfc-path) (seq nfc-path)
+ (string? nfc-path) (seq (json/decode nfc-path))))
+
+(defn- parent-field-path
+ "The nfc_path-form path of a field's parent (names from the outer-most
+ ancestor down to the parent's leaf), or nil when `parent-id` is not set. For
+ BigQuery the field's own `nfc_path` does not include the leaf field name and
+ therefore already represents the parent path; for Mongo `nfc_path` includes
+ the leaf, so the parent path is `(butlast nfc-path)`."
+ [parent-id nfc-path engine]
+ (when parent-id
+ (seq (if (= engine "bigquery-cloud-sdk")
+ nfc-path
+ (butlast nfc-path)))))
+
+(defn- format-database-id
+ "Portable id for a database — its name."
+ [db-name]
+ db-name)
+
+(defn- format-table-id
+ "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."
+ [db-name schema table-name]
+ [db-name schema table-name])
+
+(defn- format-field-id
+ "Portable id for a field — `[db schema table ...parent-path field-name]`.
+ `parent-path` may be nil (or empty) for root fields, in which case the id
+ collapses to `[db schema table field-name]`."
+ [db-name schema table-name field-name parent-path]
+ (-> [db-name schema table-name] (into parent-path) (conj field-name)))
+
+(defmulti format-entity
+ "Reshapes a raw query row for `model` into the JSON shape emitted by the
+ metadata-export endpoint."
+ {:arglists '([model row])}
+ (fn [model _row] model))
+
+(defmethod format-entity :model/Database
+ [_model {:keys [name engine]}]
+ {:name name :engine engine})
+
+(defmethod format-entity :model/Table
+ [_model {:keys [db_name schema table_name description]}]
+ (m/assoc-some {:db_id (format-database-id db_name)
+ :name table_name}
+ :schema schema
+ :description description))
+
+(defmethod format-entity :model/Field
+ [_model {:keys [db_name engine table_schema table_name field_name parent_id description
+ base_type database_type effective_type semantic_type coercion_strategy nfc_path
+ fk_db_name fk_db_engine fk_table_schema fk_table_name fk_field_name fk_parent_id
+ fk_field_nfc_path]}]
+ (let [nfc-path (decode-nfc-path nfc_path)
+ parent-path (parent-field-path parent_id nfc-path engine)
+ parent-id (when parent-path
+ (into [db_name table_schema table_name] parent-path))
+ fk-parent-path (parent-field-path fk_parent_id (decode-nfc-path fk_field_nfc_path) fk_db_engine)
+ fk-target-id (when (and fk_db_name fk_table_name fk_field_name)
+ (format-field-id fk_db_name fk_table_schema fk_table_name
+ fk_field_name fk-parent-path))]
+ (m/assoc-some {:table_id (format-table-id db_name table_schema table_name)
+ :name field_name}
+ :description description
+ :base_type base_type
+ :database_type database_type
+ :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
+ :semantic_type semantic_type
+ :coercion_strategy coercion_strategy
+ :nfc_path nfc-path
+ :parent_id parent-id
+ :fk_target_field_id fk-target-id)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj
new file mode 100644
index 00000000000..ab3e2186ca2
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.serialization.export.json
+ "Streaming JSON writers used by the metadata-export pipeline. Writers consume reducible
+ collections row-by-row so memory stays bounded regardless of result size."
+ (:require
+ [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
+
+(defn write-json-array!
+ "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded
+ directly with no transformation — apply per-row formatting via an `eduction` (or other
+ transducer pipeline) before passing in.
+
+ `run!` is required here because it dispatches through `reduce`, which consumes the
+ `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.
+ `doseq` cannot be used: it walks a seq, and producing a seq from the reducible
+ would realize every row into memory — defeating the point of streaming."
+ [^java.io.Writer writer reducible]
+ (.write writer "[")
+ (let [first? (volatile! true)]
+ (run! (fn [row]
+ (if @first?
+ (vreset! first? false)
+ (.write writer ","))
+ (json/encode-to row writer {}))
+ reducible))
+ (.write writer "]"))
+
+(defn write-json-object!
+ "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible
+ of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)
+ of already-formatted values to encode.
+
+ `run!` is used over the entries — `doseq` would walk a seq and realize the underlying
+ reducible, defeating streaming."
+ [^java.io.Writer writer entries]
+ (.write writer "{")
+ (let [first? (volatile! true)]
+ (run! (fn [[entry-name objects]]
+ (if @first? (vreset! first? false) (.write writer ","))
+ (.write writer (str "\"" entry-name "\":"))
+ (write-json-array! writer objects))
+ entries))
+ (.write writer "}"))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj
new file mode 100644
index 00000000000..0cb829c3aac
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj
@@ -0,0 +1,111 @@
+(ns metabase-enterprise.serialization.export.query
+ "Reducible queries for the metadata-export pipeline. Each model implements
+ `export-query` to produce the reducible row stream, and `export-query-filter`
+ to produce a HoneySQL where predicate scoped to a given table alias. Filters
+ are reusable across queries — e.g. the field query joins through table and
+ database, so it composes the database/table filters too."
+ (:require
+ [metabase.permissions.core :as perms]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private perm-mapping
+ "Permission mapping required to include a database/table in the export."
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})
+
+(defmulti export-query
+ "Returns a reducible row stream for `model`. `opts` carries caller context
+ (e.g. `:user-info`, section flags)."
+ {:arglists '([model opts])}
+ (fn [model _opts] model))
+
+(defmulti export-query-filter
+ "Returns a HoneySQL where predicate that restricts rows of `model` qualified by
+ table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across the
+ database/table/field hierarchy."
+ {:arglists '([model alias opts])}
+ (fn [model _alias _opts] model))
+
+(defmethod export-query-filter :model/Database
+ [_model alias {:keys [user-info]}]
+ [:and
+ [:= (u/qualified-key alias :is_audit) false]
+ [:= (u/qualified-key alias :router_database_id) nil]
+ [:in (u/qualified-key alias :id) (perms/visible-database-filter-select user-info perm-mapping)]])
+
+(defmethod export-query-filter :model/Table
+ [_model alias {:keys [user-info]}]
+ [:and
+ [:= (u/qualified-key alias :active) true]
+ [:= (u/qualified-key alias :visibility_type) nil]
+ [:in (u/qualified-key alias :id) (perms/visible-table-filter-select :id user-info perm-mapping)]])
+
+(defmethod export-query-filter :model/Field
+ [_model alias _opts]
+ [:and
+ [:= (u/qualified-key alias :active) true]
+ [:<> (u/qualified-key alias :visibility_type) "sensitive"]])
+
+(defmethod export-query :model/Database
+ [model opts]
+ (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]
+ :from [[:metabase_database :db]]
+ :where (export-query-filter model :db opts)}))
+
+(defmethod export-query :model/Table
+ [_model opts]
+ (t2/reducible-query {:select [[:db.name :db_name]
+ [:table.schema :schema]
+ [:table.name :table_name]
+ [:table.description :description]]
+ :from [[:metabase_table :table]]
+ :join [[:metabase_database :db] [:= :table.db_id :db.id]]
+ :where [:and
+ (export-query-filter :model/Database :db opts)
+ (export-query-filter :model/Table :table opts)]}))
+
+(defmethod export-query :model/Field
+ [_model opts]
+ (t2/reducible-query
+ {:select [[:db.name :db_name]
+ [:db.engine :engine]
+ [:table.schema :table_schema]
+ [:table.name :table_name]
+ [:field.name :field_name]
+ [:field.parent_id :parent_id]
+ [:field.description :description]
+ [:field.base_type :base_type]
+ [:field.database_type :database_type]
+ [:field.effective_type :effective_type]
+ [:field.semantic_type :semantic_type]
+ [:field.coercion_strategy :coercion_strategy]
+ [:field.nfc_path :nfc_path]
+ [:fk_db.name :fk_db_name]
+ [:fk_db.engine :fk_db_engine]
+ [:fk_table.schema :fk_table_schema]
+ [:fk_table.name :fk_table_name]
+ [:fk_field.name :fk_field_name]
+ [:fk_field.parent_id :fk_parent_id]
+ [:fk_field.nfc_path :fk_field_nfc_path]]
+ :from [[:metabase_field :field]]
+ :join [[:metabase_table :table] [:= :field.table_id :table.id]
+ [:metabase_database :db] [:= :table.db_id :db.id]]
+ :left-join [[:metabase_field :fk_field]
+ [:and
+ [:= :field.fk_target_field_id :fk_field.id]
+ (export-query-filter :model/Field :fk_field opts)]
+ [:metabase_table :fk_table]
+ [:and
+ [:= :fk_field.table_id :fk_table.id]
+ (export-query-filter :model/Table :fk_table opts)]
+ [:metabase_database :fk_db]
+ [:and
+ [:= :fk_table.db_id :fk_db.id]
+ (export-query-filter :model/Database :fk_db opts)]]
+ :where [:and
+ (export-query-filter :model/Database :db opts)
+ (export-query-filter :model/Table :table opts)
+ (export-query-filter :model/Field :field opts)]}))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj
deleted file mode 100644
index 22e459d4e16..00000000000
--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj
+++ /dev/null
@@ -1,187 +0,0 @@
-(ns metabase-enterprise.serialization.metadata
- "Reducible queries and row formatters that produce serdes-portable warehouse metadata
- (databases, tables, and fields) for the `/api/ee/serialization/metadata/export` endpoint.
- All references — database, table, fk_target_field — are emitted as portable references
- (names rather than numeric IDs) so the response can be ingested by another Metabase
- instance with different surrogate keys."
- (:require
- [medley.core :as m]
- [metabase.api.common :as api]
- [metabase.permissions.core :as perms]
- [metabase.util :as u]
- [metabase.util.json :as json]
- [toucan2.core :as t2]))
-
-(defn- perm-user-info
- "Current-user info passed to `perms/visible-*-filter-select` to scope the visibility filters."
- []
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*})
-
-(defn- perm-mapping
- "Permission mapping required to include a database/table in the export."
- []
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder})
-
-(defn- visible-db-where
- "Honeysql predicate selecting non-audit, non-router databases visible to the current user.
- `d` is the table alias of the joined `metabase_database`."
- [d]
- [:and
- [:= (u/qualified-key d :is_audit) false]
- [:= (u/qualified-key d :router_database_id) nil]
- [:in (u/qualified-key d :id) (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])
-
-(defn- visible-table-where
- "Honeysql predicate selecting active, non-hidden tables visible to the current user.
- `t` is the table alias of the joined `metabase_table`."
- [t]
- [:and
- [:= (u/qualified-key t :active) true]
- [:= (u/qualified-key t :visibility_type) nil]
- [:in (u/qualified-key t :id) (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])
-
-(defn- visible-field-where
- "Honeysql predicate selecting active, non-sensitive fields. `f` is the table alias of the
- joined `metabase_field`."
- [f]
- [:and
- [:= (u/qualified-key f :active) true]
- [:<> (u/qualified-key f :visibility_type) "sensitive"]])
-
-(defn- reducible-databases-query
- "Raw reducible-query streaming visible-database rows for [[reducible-databases]]."
- []
- (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]
- :from [[:metabase_database :db]]
- :where (visible-db-where :db)}))
-
-(defn- reducible-tables-query
- "Raw reducible-query streaming visible-table rows for [[reducible-tables]]."
- []
- (t2/reducible-query {:select [[:db.name :db_name]
- [:table.schema :schema]
- [:table.name :table_name]
- [:table.description :description]]
- :from [[:metabase_table :table]]
- :join [[:metabase_database :db] [:= :table.db_id :db.id]]
- :where [:and (visible-db-where :db) (visible-table-where :table)]}))
-
-(defn- reducible-fields-query
- "Raw reducible-query streaming visible-field rows for [[reducible-fields]]. Visibility
- filters on the FK target chain are folded into the LEFT JOIN ON clauses (rather than the
- WHERE) so that an inaccessible target fails the join and the `fk_*` columns come back as
- NULL — [[format-field-row]] then drops the `:fk_target_field_id` for that row."
- []
- (t2/reducible-query
- {:select [[:db.name :db_name]
- [:table.schema :table_schema]
- [:table.name :table_name]
- [:field.name :field_name]
- [:field.description :description]
- [:field.base_type :base_type]
- [:field.database_type :database_type]
- [:field.effective_type :effective_type]
- [:field.semantic_type :semantic_type]
- [:field.coercion_strategy :coercion_strategy]
- [:field.nfc_path :nfc_path]
- [:fk_db.name :fk_db_name]
- [:fk_table.schema :fk_table_schema]
- [:fk_table.name :fk_table_name]
- [:fk_field.name :fk_field_name]
- [:fk_field.nfc_path :fk_field_nfc_path]]
- :from [[:metabase_field :field]]
- :join [[:metabase_table :table] [:= :field.table_id :table.id]
- [:metabase_database :db] [:= :table.db_id :db.id]]
- :left-join [[:metabase_field :fk_field] [:and [:= :field.fk_target_field_id :fk_field.id] (visible-field-where :fk_field)]
- [:metabase_table :fk_table] [:and [:= :fk_field.table_id :fk_table.id] (visible-table-where :fk_table)]
- [:metabase_database :fk_db] [:and [:= :fk_table.db_id :fk_db.id] (visible-db-where :fk_db)]]
- :where [:and (visible-db-where :db) (visible-table-where :table) (visible-field-where :field)]}))
-
-(defn- decode-nfc-path
- "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for
- null/empty, the original value when it's already a sequential collection, or the decoded
- vector when it's a JSON string."
- [nfc-path]
- (cond
- (nil? nfc-path) nil
- (sequential? nfc-path) (seq nfc-path)
- (string? nfc-path) (seq (json/decode nfc-path))))
-
-(defn- external-database-id
- "Portable id for a database — its name."
- [db-name]
- db-name)
-
-(defn- external-table-id
- "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."
- [db-name schema table-name]
- [db-name schema table-name])
-
-(defn- external-field-id
- "Portable id for a field. When `nfc-path` is set (JSON-nested column), the path replaces
- the field's display name; the field's own name is the leaf of `nfc-path` joined with
- arrows, so the path is the canonical structural representation."
- [db-name schema table-name field-name nfc-path]
- (if (seq nfc-path)
- (into [db-name schema table-name] nfc-path)
- [db-name schema table-name field-name]))
-
-(defn format-database-row
- "JSON shape for one database row, identifying the database by its portable id."
- [{:keys [name engine]}]
- {:id (external-database-id name) :name name :engine engine})
-
-(defn format-table-row
- "JSON shape for one table row, with portable references for `:id` and `:db_id`. Optional
- fields are omitted when nil."
- [{:keys [db_name schema table_name description]}]
- (m/assoc-some {:id (external-table-id db_name schema table_name)
- :db_id (external-database-id db_name)
- :name table_name}
- :schema schema
- :description description))
-
-(defn format-field-row
- "JSON shape for one field row, with portable references for `:id`, `:table_id`, `:parent_id`,
- and `:fk_target_field_id`. `:parent_id` is derived from `nfc_path` (the portable id with the
- last path element dropped) and is only emitted when `nfc_path` has at least two elements.
- Optional fields are omitted when nil."
- [{:keys [db_name table_schema table_name field_name description base_type database_type
- effective_type semantic_type coercion_strategy nfc_path
- fk_db_name fk_table_schema fk_table_name fk_field_name fk_field_nfc_path]}]
- (let [nfc-path (decode-nfc-path nfc_path)
- parent-id (when-some [parent-nfc (seq (butlast nfc-path))]
- (external-field-id db_name table_schema table_name nil parent-nfc))
- fk-field-nfc (decode-nfc-path fk_field_nfc_path)
- fk-target-id (when (and fk_db_name fk_table_name fk_field_name)
- (external-field-id fk_db_name fk_table_schema fk_table_name
- fk_field_name fk-field-nfc))]
- (m/assoc-some {:id (external-field-id db_name table_schema table_name field_name nfc-path)
- :table_id (external-table-id db_name table_schema table_name)
- :name field_name}
- :description description
- :base_type base_type
- :database_type database_type
- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)
- :semantic_type semantic_type
- :coercion_strategy coercion_strategy
- :nfc_path nfc-path
- :parent_id parent-id
- :fk_target_field_id fk-target-id)))
-
-(defn reducible-databases
- "Eduction streaming visible databases as already-formatted JSON-shaped rows."
- []
- (eduction (map format-database-row) (reducible-databases-query)))
-
-(defn reducible-tables
- "Eduction streaming visible tables as already-formatted JSON-shaped rows."
- []
- (eduction (map format-table-row) (reducible-tables-query)))
-
-(defn reducible-fields
- "Eduction streaming visible fields as already-formatted JSON-shaped rows."
- []
- (eduction (map format-field-row) (reducible-fields-query)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
index 740ec109256..4ab457ff1bb 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj
@@ -1,9 +1,21 @@
(ns metabase-enterprise.serialization.schema
"Malli schemas for the serdes-portable shapes produced by
- [[metabase-enterprise.serialization.metadata]]."
+ [[metabase-enterprise.serialization.export.format]]."
(:require
[metabase.util.malli.registry :as mr]))
+(mr/def ::user-info
+ [:map
+ [:user-id pos-int?]
+ [:is-superuser? :boolean]])
+
+(mr/def ::export-options
+ [:map
+ [:user-info ::user-info]
+ [:with-databases {:optional true} [:maybe :boolean]]
+ [:with-tables {:optional true} [:maybe :boolean]]
+ [:with-fields {:optional true} [:maybe :boolean]]])
+
(mr/def ::external-database-id
:string)
@@ -13,23 +25,20 @@
(mr/def ::external-field-id
[:cat :string [:maybe :string] :string [:+ :string]])
-(mr/def ::database-info
+(mr/def ::external-database
[:map
- [:id ::external-database-id]
[:name :string]
[:engine :string]])
-(mr/def ::table-info
+(mr/def ::external-table
[:map
- [:id ::external-table-id]
[:db_id ::external-database-id]
[:name :string]
[:schema {:optional true} :string]
[:description {:optional true} :string]])
-(mr/def ::field-info
+(mr/def ::external-field
[:map
- [:id ::external-field-id]
[:table_id ::external-table-id]
[:name :string]
[:parent_id {:optional true} ::external-field-id]
@@ -42,8 +51,8 @@
[:coercion_strategy {:optional true} :string]
[:nfc_path {:optional true} [:sequential :string]]])
-(mr/def ::metadata-export-response
+(mr/def ::export-response
[:map
- [:databases {:optional true} [:sequential ::database-info]]
- [:tables {:optional true} [:sequential ::table-info]]
- [:fields {:optional true} [:sequential ::field-info]]])
+ [:databases {:optional true} [:sequential ::external-database]]
+ [:tables {:optional true} [:sequential ::external-table]]
+ [:fields {:optional true} [:sequential ::external-field]]])
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 56a212ef83a..5062fef9770 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -444,7 +444,7 @@
{:file (byte-array 0)})))))))
(deftest metadata-export-basic-test
- (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"
+ (testing "GET /api/ee/serialization/export/json — happy path with one db/table/field"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
@@ -454,23 +454,23 @@
:database_type "BIGINT"
:semantic_type :type/PK}]
(let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-databases true
:with-tables true
:with-fields true)
- test-db (m/find-first (comp #{db-name} :id) databases)
- test-table (m/find-first (comp #{[db-name "PUBLIC" t-name]} :id) tables)
- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]
- (is (=? {:id db-name :name db-name :engine "h2"}
+ test-db (m/find-first (comp #{db-name} :name) databases)
+ test-table (m/find-first #(and (= (:db_id %) db-name) (= (:name %) t-name)) tables)
+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
+ (= (:name %) f-name))
+ fields)]
+ (is (=? {:name db-name :engine "h2"}
test-db))
- (is (=? {:id [db-name "PUBLIC" t-name]
- :db_id db-name
+ (is (=? {:db_id db-name
:name t-name
:schema "PUBLIC"
:description "A test table"}
test-table))
- (is (=? {:id [db-name "PUBLIC" t-name f-name]
- :table_id [db-name "PUBLIC" t-name]
+ (is (=? {:table_id [db-name "PUBLIC" t-name]
:name f-name
:base_type "type/Integer"
:database_type "BIGINT"
@@ -478,7 +478,7 @@
test-field)))))))
(deftest metadata-export-optional-properties-test
- (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"
+ (testing "GET /api/ee/serialization/export/json — effective_type, coercion_strategy and description are emitted when set"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -489,11 +489,12 @@
:coercion_strategy :Coercion/ISO8601->DateTime
:description "When this happened"}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-fields true)
- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]
- (is (=? {:id [db-name "PUBLIC" t-name f-name]
- :name f-name
+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
+ (= (:name %) f-name))
+ fields)]
+ (is (=? {:name f-name
:base_type "type/Text"
:effective_type "type/DateTime"
:coercion_strategy "Coercion/ISO8601->DateTime"
@@ -501,28 +502,35 @@
test-field)))))))
(deftest metadata-export-parent-field-test
- (testing "GET /api/ee/serialization/metadata/export — parent_id is nfc_path with the last element dropped, omitted for root fields"
+ (testing "GET /api/ee/serialization/export/json — parent_id is emitted only when raw field.parent_id is set"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
:model/Field {root-name :name} {:table_id t-id
:base_type :type/Text}
- :model/Field _ {:table_id t-id
+ :model/Field {parent-id :id} {:table_id t-id
+ :name "data"
+ :base_type :type/Dictionary}
+ :model/Field _ {:table_id t-id
+ :name "city"
:base_type :type/Text
+ :parent_id parent-id
:nfc_path ["data" "city"]}]
- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ (let [table-id [db-name "PUBLIC" t-name]
+ {:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/export/json"
:with-fields true)
- test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name root-name]} :id) fields)
- test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name "data" "city"]} :id) fields)]
- (is (=? {:id [db-name "PUBLIC" t-name root-name]} test-field-1))
+ test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) root-name)) fields)
+ test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) "city")) fields)]
+ (is (=? {:name root-name :table_id table-id} test-field-1))
(is (not (contains? test-field-1 :parent_id)))
- (is (=? {:id [db-name "PUBLIC" t-name "data" "city"]
+ (is (=? {:name "city"
+ :table_id table-id
:parent_id [db-name "PUBLIC" t-name "data"]}
test-field-2)))))))
(deftest metadata-export-fk-test
- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted in portable form"
+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is emitted in portable form"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -533,21 +541,22 @@
:base_type :type/Integer
:semantic_type :type/FK
:fk_target_field_id pk-id}]
- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ (let [table-id [db-name "PUBLIC" t-name]
+ {:keys [fields]} (mt/user-http-request :crowberto :get 202
+ "ee/serialization/export/json"
:with-fields true)
- test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name pk-name]} :id) fields)
- test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]
- (is (=? {:id [db-name "PUBLIC" t-name pk-name] :semantic_type "type/PK"}
+ test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) pk-name)) fields)
+ test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) fk-name)) fields)]
+ (is (=? {:name pk-name :table_id table-id :semantic_type "type/PK"}
test-field-1))
- (is (=? {:id [db-name "PUBLIC" t-name fk-name]
- :table_id [db-name "PUBLIC" t-name]
+ (is (=? {:name fk-name
+ :table_id table-id
:semantic_type "type/FK"
- :fk_target_field_id (:id test-field-1)}
+ :fk_target_field_id [db-name "PUBLIC" t-name pk-name]}
test-field-2)))))))
(deftest metadata-export-hidden-table-test
- (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"
+ (testing "GET /api/ee/serialization/export/json — hidden tables and their fields are excluded"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
:model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -558,7 +567,7 @@
:model/Field {hidden-f-name :name} {:table_id hidden-id
:base_type :type/Integer}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-tables true :with-fields true)]
(is (some (comp #{visible-t-name} :name) tables))
(is (not-any? (comp #{hidden-t-name} :name) tables))
@@ -566,7 +575,7 @@
(is (not-any? (comp #{hidden-f-name} :name) fields)))))))
(deftest metadata-export-hidden-table-fk-test
- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is hidden"
+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is hidden"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -580,14 +589,16 @@
:semantic_type :type/FK
:fk_target_field_id target-id}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-fields true)
- test-field (m/find-first (comp #{[db-name "PUBLIC" v-name fk-name]} :id) fields)]
+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" v-name])
+ (= (:name %) fk-name))
+ fields)]
(is (some? test-field))
(is (not (contains? test-field :fk_target_field_id))))))))
(deftest metadata-export-inactive-table-test
- (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"
+ (testing "GET /api/ee/serialization/export/json — inactive tables and all of their fields are excluded"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"
@@ -598,14 +609,14 @@
:base_type :type/Integer
:active false}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-tables true :with-fields true)]
(is (not-any? (comp #{t-name} :name) tables))
(is (not-any? (comp #{active-f-name} :name) fields))
(is (not-any? (comp #{inactive-f-name} :name) fields)))))))
(deftest metadata-export-sensitive-field-test
- (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"
+ (testing "GET /api/ee/serialization/export/json — sensitive fields are excluded"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
:model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}
@@ -613,13 +624,13 @@
:model/Field {secret-name :name} {:table_id t-id :base_type :type/Text
:visibility_type :sensitive}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-fields true)]
(is (some (comp #{email-name} :name) fields))
(is (not-any? (comp #{secret-name} :name) fields)))))))
(deftest metadata-export-sensitive-fk-test
- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target field is sensitive"
+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target field is sensitive"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -631,14 +642,16 @@
:semantic_type :type/FK
:fk_target_field_id target-id}]
(let [{:keys [fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-fields true)
- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]
+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])
+ (= (:name %) fk-name))
+ fields)]
(is (some? test-field))
(is (not (contains? test-field :fk_target_field_id))))))))
(deftest metadata-export-inactive-field-test
- (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"
+ (testing "GET /api/ee/serialization/export/json — inactive fields are excluded; their active sibling and table still appear"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -648,27 +661,27 @@
:base_type :type/Integer
:active false}]
(let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-tables true :with-fields true)]
(is (some (comp #{t-name} :name) tables))
(is (some (comp #{active-f-name} :name) fields))
(is (not-any? (comp #{inactive-f-name} :name) fields)))))))
(deftest metadata-export-db-routing-test
- (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"
+ (testing "GET /api/ee/serialization/export/json — router (mirror) databases are excluded"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database _ {:engine :h2}
:model/Database {primary-id :id primary-name :name} {:engine :h2}
:model/Database {mirror-name :name} {:engine :h2
:router_database_id primary-id}]
(let [{:keys [databases]} (mt/user-http-request :crowberto :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-databases true)]
(is (some (comp #{primary-name} :name) databases))
(is (not-any? (comp #{mirror-name} :name) databases)))))))
(deftest metadata-export-table-permission-test
- (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"
+ (testing "GET /api/ee/serialization/export/json — tables the current user can't access are excluded"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}
:model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -685,7 +698,7 @@
(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
(let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-tables true :with-fields true)]
(is (some (comp #{accessible-name} :name) tables))
(is (not-any? (comp #{restricted-name} :name) tables))
@@ -693,7 +706,7 @@
(is (not-any? (comp #{no-f-name} :name) fields)))))))
(deftest metadata-export-table-permission-fk-test
- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is not accessible"
+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is not accessible"
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}
:model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}
@@ -713,18 +726,20 @@
(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)
(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)
(let [{:keys [fields]} (mt/user-http-request :rasta :get 202
- "ee/serialization/metadata/export"
+ "ee/serialization/export/json"
:with-fields true)
- test-field (m/find-first (comp #{[db-name "PUBLIC" accessible-name fk-name]} :id) fields)]
+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" accessible-name])
+ (= (:name %) fk-name))
+ fields)]
(is (some? test-field))
(is (not (contains? test-field :fk_target_field_id))))))))
(deftest metadata-export-token-feature-test
- (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"
+ (testing "GET /api/ee/serialization/export/json requires the :serialization premium feature"
(mt/with-premium-features #{}
(mt/assert-has-premium-feature-error
"Serialization"
- (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))
+ (mt/user-http-request :crowberto :get 402 "ee/serialization/export/json")))))
(deftest serialization-cleanup-test
(testing "No temp files are left behind after export/import operations"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj
new file mode 100644
index 00000000000..df4ae776485
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj
@@ -0,0 +1,156 @@
+(ns metabase-enterprise.serialization.export.format-test
+ "Unit tests for `format-entity` in
+ `metabase-enterprise.serialization.export.format`. Each deftest exercises a
+ single case so that failures pinpoint the exact behavior that broke."
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase-enterprise.serialization.export.format :as export.format]))
+
+(defn- mock-database
+ "Default :model/Database query row, with per-test overrides merged in."
+ [overrides]
+ (merge {:name "Sample" :engine "h2"} overrides))
+
+(defn- mock-table
+ "Default :model/Table query row, with per-test overrides merged in."
+ [overrides]
+ (merge {:db_name "Sample"
+ :schema "PUBLIC"
+ :table_name "Orders"
+ :description nil}
+ overrides))
+
+(defn- mock-field
+ "Default :model/Field query row, with per-test overrides merged in. Covers
+ every key destructured by `format-entity :model/Field`, so tests only need to
+ set the fields relevant to what they're checking."
+ [overrides]
+ (merge {:db_name "Sample"
+ :engine "h2"
+ :table_schema "PUBLIC"
+ :table_name "Orders"
+ :field_name "id"
+ :parent_id nil
+ :description nil
+ :base_type "type/Integer"
+ :database_type nil
+ :effective_type nil
+ :semantic_type nil
+ :coercion_strategy nil
+ :nfc_path nil
+ :fk_db_name nil
+ :fk_db_engine nil
+ :fk_table_schema nil
+ :fk_table_name nil
+ :fk_field_name nil
+ :fk_parent_id nil
+ :fk_field_nfc_path nil}
+ overrides))
+
+(deftest format-database-test
+ (is (=? {:name "Sample" :engine "h2"}
+ (export.format/format-entity :model/Database (mock-database {})))))
+
+(deftest format-table-with-schema-and-description-test
+ (is (=? {:db_id "Sample"
+ :name "Orders"
+ :schema "PUBLIC"
+ :description "Orders table"}
+ (export.format/format-entity :model/Table
+ (mock-table {:description "Orders table"})))))
+
+(deftest format-table-without-schema-test
+ (let [out (export.format/format-entity :model/Table (mock-table {:schema nil}))]
+ (is (=? {:db_id "Sample" :name "Orders"} out))
+ (is (not (contains? out :schema)))
+ (is (not (contains? out :description)))))
+
+(deftest format-field-root-test
+ (let [out (export.format/format-entity :model/Field (mock-field {}))]
+ (is (=? {:table_id ["Sample" "PUBLIC" "Orders"]
+ :name "id"
+ :base_type "type/Integer"}
+ out))
+ (is (not (contains? out :parent_id)))
+ (is (not (contains? out :fk_target_field_id)))))
+
+(deftest format-field-mongo-nested-parent-id-test
+ (is (=? {:name "city"
+ :parent_id ["Sample" "PUBLIC" "Orders" "data"]}
+ (export.format/format-entity :model/Field
+ (mock-field {:engine "mongo"
+ :field_name "city"
+ :parent_id 42
+ :base_type "type/Text"
+ :nfc_path ["data" "city"]})))))
+
+(deftest format-field-bigquery-nested-parent-id-test
+ (is (=? {:name "city"
+ :parent_id ["Sample" nil "orders" "data"]}
+ (export.format/format-entity :model/Field
+ (mock-field {:engine "bigquery-cloud-sdk"
+ :table_schema nil
+ :table_name "orders"
+ :field_name "city"
+ :parent_id 42
+ :base_type "type/Text"
+ :nfc_path ["data"]})))))
+
+(deftest format-field-no-parent-id-when-raw-parent-id-missing-test
+ (let [out (export.format/format-entity :model/Field
+ (mock-field {:engine "mongo"
+ :field_name "city"
+ :base_type "type/Text"
+ :nfc_path ["data" "city"]}))]
+ (is (not (contains? out :parent_id)))))
+
+(deftest format-field-fk-target-root-test
+ (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "id"]}
+ (export.format/format-entity :model/Field
+ (mock-field {:field_name "user_id"
+ :fk_db_name "Sample"
+ :fk_db_engine "h2"
+ :fk_table_schema "PUBLIC"
+ :fk_table_name "Orders"
+ :fk_field_name "id"})))))
+
+(deftest format-field-fk-target-mongo-nested-test
+ (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "data" "city"]}
+ (export.format/format-entity :model/Field
+ (mock-field {:engine "mongo"
+ :table_name "Customers"
+ :field_name "city_ref"
+ :base_type "type/Text"
+ :fk_db_name "Sample"
+ :fk_db_engine "mongo"
+ :fk_table_schema "PUBLIC"
+ :fk_table_name "Orders"
+ :fk_field_name "city"
+ :fk_parent_id 99
+ :fk_field_nfc_path ["data" "city"]})))))
+
+(deftest format-field-fk-target-bigquery-nested-test
+ (is (=? {:fk_target_field_id ["Sample" nil "orders" "data" "city"]}
+ (export.format/format-entity :model/Field
+ (mock-field {:table_name "Customers"
+ :field_name "city_ref"
+ :base_type "type/Text"
+ :fk_db_name "Sample"
+ :fk_db_engine "bigquery-cloud-sdk"
+ :fk_table_schema nil
+ :fk_table_name "orders"
+ :fk_field_name "city"
+ :fk_parent_id 99
+ :fk_field_nfc_path ["data"]})))))
+
+(deftest format-field-effective-type-redundant-with-base-type-test
+ (let [out (export.format/format-entity :model/Field
+ (mock-field {:effective_type "type/Integer"}))]
+ (is (not (contains? out :effective_type)))))
+
+(deftest format-field-effective-type-distinct-from-base-type-test
+ (is (=? {:effective_type "type/Date"}
+ (export.format/format-entity :model/Field
+ (mock-field {:field_name "created_at"
+ :base_type "type/Text"
+ :effective_type "type/Date"})))))