The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add clear per-model slots for json-based serialization (#73851)
Alexander Polyankin83547976da39
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 32b3e195a77..6b3e74ae9b0 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -3,7 +3,7 @@[clojure.java.io :as io][clojure.string :as str][java-time.api :as t]- [metabase-enterprise.serialization.metadata :as metadata]+ [metabase-enterprise.serialization.export :as export][metabase-enterprise.serialization.schema :as schema][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest]@@ -22,7 +22,6 @@[metabase.util :as u][metabase.util.compress :as u.compress][metabase.util.date-2 :as u.date]- [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -309,67 +308,10 @@(finally(io/delete-file (:tempfile file)))))-;;; ----------------------------------- GET /api/ee/serialization/metadata/export -----------------------------------+;;; ----------------------------------- GET /api/ee/serialization/export/json -------------------------------------(defn- write-json-array!- "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded- directly with no transformation — apply per-row formatting via an `eduction` (or other- transducer pipeline) before passing in.-- `run!` is required here because it dispatches through `reduce`, which consumes the- `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.- `doseq` cannot be used: it walks a seq, and producing a seq from the reducible- would realize every row into memory — defeating the point of streaming."- [^java.io.Writer writer reducible]- (.write writer "[")- (let [first? (volatile! true)]- (run! (fn [row]- (if @first?- (vreset! first? false)- (.write writer ","))- (json/encode-to row writer {}))- reducible))- (.write writer "]"))--(defn- write-json-object!- "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible- of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)- of already-formatted values to encode.-- `run!` is used over the entries — `doseq` would walk a seq and realize the underlying- reducible, defeating streaming."- [^java.io.Writer writer entries]- (.write writer "{")- (let [first? (volatile! true)]- (run! (fn [[entry-name objects]]- (if @first? (vreset! first? false) (.write writer ","))- (.write writer (str "\"" entry-name "\":"))- (write-json-array! writer objects))- entries))- (.write writer "}"))--(defn- write-databases-metadata!- "Streams the databases/tables/fields metadata to the given OutputStream. Sections are- included only when their `with-...?` flag is true. All references — database, table,- fk_target_field — are emitted in serdes-portable form (names rather than numeric IDs)- so the response can be ingested by another Metabase instance with different surrogate- keys.-- Warehouses with large schemas can produce gigabytes of metadata, so rows are pulled- from reducible queries and streamed directly to the writer — memory stays bounded- regardless of schema size."- [^java.io.OutputStream os {:keys [with-databases? with-tables? with-fields?]}]- (let [writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. os java.nio.charset.StandardCharsets/UTF_8))]- (write-json-object!- writer- (cond-> []- with-databases? (conj ["databases" (metadata/reducible-databases)])- with-tables? (conj ["tables" (metadata/reducible-tables)])- with-fields? (conj ["fields" (metadata/reducible-fields)])))- (.flush writer)))--(api.macros/defendpoint :get "/metadata/export"- :- (sr/streaming-response-schema ::schema/metadata-export-response)+(api.macros/defendpoint :get "/export/json"+ :- (sr/streaming-response-schema ::schema/export-response)"Get warehouse metadata (databases, tables, and fields) for all databases visible to thecurrent user, with all references emitted in serdes-portable form (database names, table`[db schema name]` tuples, field `[db schema table name | nfc-path...]` tuples).@@ -381,16 +323,14 @@Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or`Query builder and native`) permissions on each database and table."[_route-params- {:keys [with-databases with-tables with-fields]}- :- [:map- [:with-databases {:default false} [:maybe :boolean]]- [:with-tables {:default false} [:maybe :boolean]]- [:with-fields {:default false} [:maybe :boolean]]]]- (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]- (write-databases-metadata! os- {:with-databases? with-databases- :with-tables? with-tables- :with-fields? with-fields})))+ query-params :- [:map+ [:with-databases {:default false} [:maybe :boolean]]+ [:with-tables {:default false} [:maybe :boolean]]+ [:with-fields {:default false} [:maybe :boolean]]]]+ (let [opts (assoc query-params :user-info {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*})]+ (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]+ (export/export! os opts))))(def ^{:arglists '([request respond raise])} routes"`/api/ee/serialization` routes."diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export.clj b/enterprise/backend/src/metabase_enterprise/serialization/export.cljnew file mode 100644index 00000000000..06bf27275da--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/export.clj@@ -0,0 +1,43 @@+(ns metabase-enterprise.serialization.export+ "Orchestrates the metadata-export pipeline: walks [[sections]], runs each+ section's `export-query`, formats rows via `format-entity`, and streams the+ result as a JSON object whose keys are the section names. Sections whose+ `:enabled?` predicate returns false for `opts` are skipped."+ (:require+ [metabase-enterprise.serialization.export.format :as export.format]+ [metabase-enterprise.serialization.export.json :as export.json]+ [metabase-enterprise.serialization.export.query :as export.query]+ [metabase-enterprise.serialization.schema :as schema]+ [metabase.util.malli :as mu])+ (:import+ (java.io BufferedWriter OutputStream OutputStreamWriter)+ (java.nio.charset StandardCharsets)))++(set! *warn-on-reflection* true)++(def sections+ "Sections emitted by the metadata export, in response order."+ [{:name "databases" :model :model/Database :enabled? :with-databases}+ {:name "tables" :model :model/Table :enabled? :with-tables}+ {:name "fields" :model :model/Field :enabled? :with-fields}])++(mu/defn export!+ "Streams the metadata export to `os` as a JSON object. Walks [[sections]] and+ includes each section only when its `:enabled?` predicate is truthy for `opts`.++ Warehouses with large schemas can produce gigabytes of metadata, so rows are+ pulled from reducible queries and streamed directly to the writer — memory+ stays bounded regardless of schema size."+ [^OutputStream os+ opts :- ::schema/export-options]+ (let [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]+ (export.json/write-json-object!+ writer+ (eduction+ (filter (fn [{:keys [enabled?]}] (enabled? opts)))+ (map (fn [{:keys [name model]}]+ [name (eduction+ (map #(export.format/format-entity model %))+ (export.query/export-query model opts))]))+ sections))+ (.flush writer)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/format.cljnew file mode 100644index 00000000000..769df864de6--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/format.clj@@ -0,0 +1,93 @@+(ns metabase-enterprise.serialization.export.format+ "Per-model row formatters for the metadata-export pipeline. Each row coming out+ of the export query is reshaped via `format-entity` into the JSON shape the+ endpoint emits, with all references — database, table, fk_target_field —+ rendered in serdes-portable form (names rather than numeric IDs) so the+ response can be ingested by another Metabase instance with different+ surrogate keys."+ (:require+ [medley.core :as m]+ [metabase.util.json :as json]))++(set! *warn-on-reflection* true)++(defn- decode-nfc-path+ "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for+ null/empty, the original value when it's already a sequential collection, or the decoded+ vector when it's a JSON string."+ [nfc-path]+ (cond+ (nil? nfc-path) nil+ (sequential? nfc-path) (seq nfc-path)+ (string? nfc-path) (seq (json/decode nfc-path))))++(defn- parent-field-path+ "The nfc_path-form path of a field's parent (names from the outer-most+ ancestor down to the parent's leaf), or nil when `parent-id` is not set. For+ BigQuery the field's own `nfc_path` does not include the leaf field name and+ therefore already represents the parent path; for Mongo `nfc_path` includes+ the leaf, so the parent path is `(butlast nfc-path)`."+ [parent-id nfc-path engine]+ (when parent-id+ (seq (if (= engine "bigquery-cloud-sdk")+ nfc-path+ (butlast nfc-path)))))++(defn- format-database-id+ "Portable id for a database — its name."+ [db-name]+ db-name)++(defn- format-table-id+ "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."+ [db-name schema table-name]+ [db-name schema table-name])++(defn- format-field-id+ "Portable id for a field — `[db schema table ...parent-path field-name]`.+ `parent-path` may be nil (or empty) for root fields, in which case the id+ collapses to `[db schema table field-name]`."+ [db-name schema table-name field-name parent-path]+ (-> [db-name schema table-name] (into parent-path) (conj field-name)))++(defmulti format-entity+ "Reshapes a raw query row for `model` into the JSON shape emitted by the+ metadata-export endpoint."+ {:arglists '([model row])}+ (fn [model _row] model))++(defmethod format-entity :model/Database+ [_model {:keys [name engine]}]+ {:name name :engine engine})++(defmethod format-entity :model/Table+ [_model {:keys [db_name schema table_name description]}]+ (m/assoc-some {:db_id (format-database-id db_name)+ :name table_name}+ :schema schema+ :description description))++(defmethod format-entity :model/Field+ [_model {:keys [db_name engine table_schema table_name field_name parent_id description+ base_type database_type effective_type semantic_type coercion_strategy nfc_path+ fk_db_name fk_db_engine fk_table_schema fk_table_name fk_field_name fk_parent_id+ fk_field_nfc_path]}]+ (let [nfc-path (decode-nfc-path nfc_path)+ parent-path (parent-field-path parent_id nfc-path engine)+ parent-id (when parent-path+ (into [db_name table_schema table_name] parent-path))+ fk-parent-path (parent-field-path fk_parent_id (decode-nfc-path fk_field_nfc_path) fk_db_engine)+ fk-target-id (when (and fk_db_name fk_table_name fk_field_name)+ (format-field-id fk_db_name fk_table_schema fk_table_name+ fk_field_name fk-parent-path))]+ (m/assoc-some {:table_id (format-table-id db_name table_schema table_name)+ :name field_name}+ :description description+ :base_type base_type+ :database_type database_type+ :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)+ :semantic_type semantic_type+ :coercion_strategy coercion_strategy+ :nfc_path nfc-path+ :parent_id parent-id+ :fk_target_field_id fk-target-id)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/json.cljnew file mode 100644index 00000000000..ab3e2186ca2--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/json.clj@@ -0,0 +1,44 @@+(ns metabase-enterprise.serialization.export.json+ "Streaming JSON writers used by the metadata-export pipeline. Writers consume reducible+ collections row-by-row so memory stays bounded regardless of result size."+ (:require+ [metabase.util.json :as json]))++(set! *warn-on-reflection* true)++(defn write-json-array!+ "Streams a reducible collection as a JSON array to `writer`. Each value is JSON-encoded+ directly with no transformation — apply per-row formatting via an `eduction` (or other+ transducer pipeline) before passing in.++ `run!` is required here because it dispatches through `reduce`, which consumes the+ `IReduceInit` returned by `t2/reducible-query` row-by-row without materializing.+ `doseq` cannot be used: it walks a seq, and producing a seq from the reducible+ would realize every row into memory — defeating the point of streaming."+ [^java.io.Writer writer reducible]+ (.write writer "[")+ (let [first? (volatile! true)]+ (run! (fn [row]+ (if @first?+ (vreset! first? false)+ (.write writer ","))+ (json/encode-to row writer {}))+ reducible))+ (.write writer "]"))++(defn write-json-object!+ "Writes a JSON object whose values are JSON arrays to `writer`. `entries` is a reducible+ of `[entry-name objects]` pairs; `objects` is itself a reducible (typically an `eduction`)+ of already-formatted values to encode.++ `run!` is used over the entries — `doseq` would walk a seq and realize the underlying+ reducible, defeating streaming."+ [^java.io.Writer writer entries]+ (.write writer "{")+ (let [first? (volatile! true)]+ (run! (fn [[entry-name objects]]+ (if @first? (vreset! first? false) (.write writer ","))+ (.write writer (str "\"" entry-name "\":"))+ (write-json-array! writer objects))+ entries))+ (.write writer "}"))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj b/enterprise/backend/src/metabase_enterprise/serialization/export/query.cljnew file mode 100644index 00000000000..0cb829c3aac--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/export/query.clj@@ -0,0 +1,111 @@+(ns metabase-enterprise.serialization.export.query+ "Reducible queries for the metadata-export pipeline. Each model implements+ `export-query` to produce the reducible row stream, and `export-query-filter`+ to produce a HoneySQL where predicate scoped to a given table alias. Filters+ are reusable across queries — e.g. the field query joins through table and+ database, so it composes the database/table filters too."+ (:require+ [metabase.permissions.core :as perms]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private perm-mapping+ "Permission mapping required to include a database/table in the export."+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})++(defmulti export-query+ "Returns a reducible row stream for `model`. `opts` carries caller context+ (e.g. `:user-info`, section flags)."+ {:arglists '([model opts])}+ (fn [model _opts] model))++(defmulti export-query-filter+ "Returns a HoneySQL where predicate that restricts rows of `model` qualified by+ table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across the+ database/table/field hierarchy."+ {:arglists '([model alias opts])}+ (fn [model _alias _opts] model))++(defmethod export-query-filter :model/Database+ [_model alias {:keys [user-info]}]+ [:and+ [:= (u/qualified-key alias :is_audit) false]+ [:= (u/qualified-key alias :router_database_id) nil]+ [:in (u/qualified-key alias :id) (perms/visible-database-filter-select user-info perm-mapping)]])++(defmethod export-query-filter :model/Table+ [_model alias {:keys [user-info]}]+ [:and+ [:= (u/qualified-key alias :active) true]+ [:= (u/qualified-key alias :visibility_type) nil]+ [:in (u/qualified-key alias :id) (perms/visible-table-filter-select :id user-info perm-mapping)]])++(defmethod export-query-filter :model/Field+ [_model alias _opts]+ [:and+ [:= (u/qualified-key alias :active) true]+ [:<> (u/qualified-key alias :visibility_type) "sensitive"]])++(defmethod export-query :model/Database+ [model opts]+ (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]+ :from [[:metabase_database :db]]+ :where (export-query-filter model :db opts)}))++(defmethod export-query :model/Table+ [_model opts]+ (t2/reducible-query {:select [[:db.name :db_name]+ [:table.schema :schema]+ [:table.name :table_name]+ [:table.description :description]]+ :from [[:metabase_table :table]]+ :join [[:metabase_database :db] [:= :table.db_id :db.id]]+ :where [:and+ (export-query-filter :model/Database :db opts)+ (export-query-filter :model/Table :table opts)]}))++(defmethod export-query :model/Field+ [_model opts]+ (t2/reducible-query+ {:select [[:db.name :db_name]+ [:db.engine :engine]+ [:table.schema :table_schema]+ [:table.name :table_name]+ [:field.name :field_name]+ [:field.parent_id :parent_id]+ [:field.description :description]+ [:field.base_type :base_type]+ [:field.database_type :database_type]+ [:field.effective_type :effective_type]+ [:field.semantic_type :semantic_type]+ [:field.coercion_strategy :coercion_strategy]+ [:field.nfc_path :nfc_path]+ [:fk_db.name :fk_db_name]+ [:fk_db.engine :fk_db_engine]+ [:fk_table.schema :fk_table_schema]+ [:fk_table.name :fk_table_name]+ [:fk_field.name :fk_field_name]+ [:fk_field.parent_id :fk_parent_id]+ [:fk_field.nfc_path :fk_field_nfc_path]]+ :from [[:metabase_field :field]]+ :join [[:metabase_table :table] [:= :field.table_id :table.id]+ [:metabase_database :db] [:= :table.db_id :db.id]]+ :left-join [[:metabase_field :fk_field]+ [:and+ [:= :field.fk_target_field_id :fk_field.id]+ (export-query-filter :model/Field :fk_field opts)]+ [:metabase_table :fk_table]+ [:and+ [:= :fk_field.table_id :fk_table.id]+ (export-query-filter :model/Table :fk_table opts)]+ [:metabase_database :fk_db]+ [:and+ [:= :fk_table.db_id :fk_db.id]+ (export-query-filter :model/Database :fk_db opts)]]+ :where [:and+ (export-query-filter :model/Database :db opts)+ (export-query-filter :model/Table :table opts)+ (export-query-filter :model/Field :field opts)]}))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata.cljdeleted file mode 100644index 22e459d4e16..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata.clj+++ /dev/null@@ -1,187 +0,0 @@-(ns metabase-enterprise.serialization.metadata- "Reducible queries and row formatters that produce serdes-portable warehouse metadata- (databases, tables, and fields) for the `/api/ee/serialization/metadata/export` endpoint.- All references — database, table, fk_target_field — are emitted as portable references- (names rather than numeric IDs) so the response can be ingested by another Metabase- instance with different surrogate keys."- (:require- [medley.core :as m]- [metabase.api.common :as api]- [metabase.permissions.core :as perms]- [metabase.util :as u]- [metabase.util.json :as json]- [toucan2.core :as t2]))--(defn- perm-user-info- "Current-user info passed to `perms/visible-*-filter-select` to scope the visibility filters."- []- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*})--(defn- perm-mapping- "Permission mapping required to include a database/table in the export."- []- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})--(defn- visible-db-where- "Honeysql predicate selecting non-audit, non-router databases visible to the current user.- `d` is the table alias of the joined `metabase_database`."- [d]- [:and- [:= (u/qualified-key d :is_audit) false]- [:= (u/qualified-key d :router_database_id) nil]- [:in (u/qualified-key d :id) (perms/visible-database-filter-select (perm-user-info) (perm-mapping))]])--(defn- visible-table-where- "Honeysql predicate selecting active, non-hidden tables visible to the current user.- `t` is the table alias of the joined `metabase_table`."- [t]- [:and- [:= (u/qualified-key t :active) true]- [:= (u/qualified-key t :visibility_type) nil]- [:in (u/qualified-key t :id) (perms/visible-table-filter-select :id (perm-user-info) (perm-mapping))]])--(defn- visible-field-where- "Honeysql predicate selecting active, non-sensitive fields. `f` is the table alias of the- joined `metabase_field`."- [f]- [:and- [:= (u/qualified-key f :active) true]- [:<> (u/qualified-key f :visibility_type) "sensitive"]])--(defn- reducible-databases-query- "Raw reducible-query streaming visible-database rows for [[reducible-databases]]."- []- (t2/reducible-query {:select [[:db.name :name] [:db.engine :engine]]- :from [[:metabase_database :db]]- :where (visible-db-where :db)}))--(defn- reducible-tables-query- "Raw reducible-query streaming visible-table rows for [[reducible-tables]]."- []- (t2/reducible-query {:select [[:db.name :db_name]- [:table.schema :schema]- [:table.name :table_name]- [:table.description :description]]- :from [[:metabase_table :table]]- :join [[:metabase_database :db] [:= :table.db_id :db.id]]- :where [:and (visible-db-where :db) (visible-table-where :table)]}))--(defn- reducible-fields-query- "Raw reducible-query streaming visible-field rows for [[reducible-fields]]. Visibility- filters on the FK target chain are folded into the LEFT JOIN ON clauses (rather than the- WHERE) so that an inaccessible target fails the join and the `fk_*` columns come back as- NULL — [[format-field-row]] then drops the `:fk_target_field_id` for that row."- []- (t2/reducible-query- {:select [[:db.name :db_name]- [:table.schema :table_schema]- [:table.name :table_name]- [:field.name :field_name]- [:field.description :description]- [:field.base_type :base_type]- [:field.database_type :database_type]- [:field.effective_type :effective_type]- [:field.semantic_type :semantic_type]- [:field.coercion_strategy :coercion_strategy]- [:field.nfc_path :nfc_path]- [:fk_db.name :fk_db_name]- [:fk_table.schema :fk_table_schema]- [:fk_table.name :fk_table_name]- [:fk_field.name :fk_field_name]- [:fk_field.nfc_path :fk_field_nfc_path]]- :from [[:metabase_field :field]]- :join [[:metabase_table :table] [:= :field.table_id :table.id]- [:metabase_database :db] [:= :table.db_id :db.id]]- :left-join [[:metabase_field :fk_field] [:and [:= :field.fk_target_field_id :fk_field.id] (visible-field-where :fk_field)]- [:metabase_table :fk_table] [:and [:= :fk_field.table_id :fk_table.id] (visible-table-where :fk_table)]- [:metabase_database :fk_db] [:and [:= :fk_table.db_id :fk_db.id] (visible-db-where :fk_db)]]- :where [:and (visible-db-where :db) (visible-table-where :table) (visible-field-where :field)]}))--(defn- decode-nfc-path- "JSON-decode an `nfc_path` value pulled via raw query (no model transform). Returns nil for- null/empty, the original value when it's already a sequential collection, or the decoded- vector when it's a JSON string."- [nfc-path]- (cond- (nil? nfc-path) nil- (sequential? nfc-path) (seq nfc-path)- (string? nfc-path) (seq (json/decode nfc-path))))--(defn- external-database-id- "Portable id for a database — its name."- [db-name]- db-name)--(defn- external-table-id- "Portable id for a table — `[db-name schema table-name]` (schema may be nil)."- [db-name schema table-name]- [db-name schema table-name])--(defn- external-field-id- "Portable id for a field. When `nfc-path` is set (JSON-nested column), the path replaces- the field's display name; the field's own name is the leaf of `nfc-path` joined with- arrows, so the path is the canonical structural representation."- [db-name schema table-name field-name nfc-path]- (if (seq nfc-path)- (into [db-name schema table-name] nfc-path)- [db-name schema table-name field-name]))--(defn format-database-row- "JSON shape for one database row, identifying the database by its portable id."- [{:keys [name engine]}]- {:id (external-database-id name) :name name :engine engine})--(defn format-table-row- "JSON shape for one table row, with portable references for `:id` and `:db_id`. Optional- fields are omitted when nil."- [{:keys [db_name schema table_name description]}]- (m/assoc-some {:id (external-table-id db_name schema table_name)- :db_id (external-database-id db_name)- :name table_name}- :schema schema- :description description))--(defn format-field-row- "JSON shape for one field row, with portable references for `:id`, `:table_id`, `:parent_id`,- and `:fk_target_field_id`. `:parent_id` is derived from `nfc_path` (the portable id with the- last path element dropped) and is only emitted when `nfc_path` has at least two elements.- Optional fields are omitted when nil."- [{:keys [db_name table_schema table_name field_name description base_type database_type- effective_type semantic_type coercion_strategy nfc_path- fk_db_name fk_table_schema fk_table_name fk_field_name fk_field_nfc_path]}]- (let [nfc-path (decode-nfc-path nfc_path)- parent-id (when-some [parent-nfc (seq (butlast nfc-path))]- (external-field-id db_name table_schema table_name nil parent-nfc))- fk-field-nfc (decode-nfc-path fk_field_nfc_path)- fk-target-id (when (and fk_db_name fk_table_name fk_field_name)- (external-field-id fk_db_name fk_table_schema fk_table_name- fk_field_name fk-field-nfc))]- (m/assoc-some {:id (external-field-id db_name table_schema table_name field_name nfc-path)- :table_id (external-table-id db_name table_schema table_name)- :name field_name}- :description description- :base_type base_type- :database_type database_type- :effective_type (when (and effective_type (not= base_type effective_type)) effective_type)- :semantic_type semantic_type- :coercion_strategy coercion_strategy- :nfc_path nfc-path- :parent_id parent-id- :fk_target_field_id fk-target-id)))--(defn reducible-databases- "Eduction streaming visible databases as already-formatted JSON-shaped rows."- []- (eduction (map format-database-row) (reducible-databases-query)))--(defn reducible-tables- "Eduction streaming visible tables as already-formatted JSON-shaped rows."- []- (eduction (map format-table-row) (reducible-tables-query)))--(defn reducible-fields- "Eduction streaming visible fields as already-formatted JSON-shaped rows."- []- (eduction (map format-field-row) (reducible-fields-query)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.cljindex 740ec109256..4ab457ff1bb 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/schema.clj@@ -1,9 +1,21 @@(ns metabase-enterprise.serialization.schema"Malli schemas for the serdes-portable shapes produced by- [[metabase-enterprise.serialization.metadata]]."+ [[metabase-enterprise.serialization.export.format]]."(:require[metabase.util.malli.registry :as mr]))+(mr/def ::user-info+ [:map+ [:user-id pos-int?]+ [:is-superuser? :boolean]])++(mr/def ::export-options+ [:map+ [:user-info ::user-info]+ [:with-databases {:optional true} [:maybe :boolean]]+ [:with-tables {:optional true} [:maybe :boolean]]+ [:with-fields {:optional true} [:maybe :boolean]]])+(mr/def ::external-database-id:string)@@ -13,23 +25,20 @@(mr/def ::external-field-id[:cat :string [:maybe :string] :string [:+ :string]])-(mr/def ::database-info+(mr/def ::external-database[:map- [:id ::external-database-id][:name :string][:engine :string]])-(mr/def ::table-info+(mr/def ::external-table[:map- [:id ::external-table-id][:db_id ::external-database-id][:name :string][:schema {:optional true} :string][:description {:optional true} :string]])-(mr/def ::field-info+(mr/def ::external-field[:map- [:id ::external-field-id][:table_id ::external-table-id][:name :string][:parent_id {:optional true} ::external-field-id]@@ -42,8 +51,8 @@[:coercion_strategy {:optional true} :string][:nfc_path {:optional true} [:sequential :string]]])-(mr/def ::metadata-export-response+(mr/def ::export-response[:map- [:databases {:optional true} [:sequential ::database-info]]- [:tables {:optional true} [:sequential ::table-info]]- [:fields {:optional true} [:sequential ::field-info]]])+ [:databases {:optional true} [:sequential ::external-database]]+ [:tables {:optional true} [:sequential ::external-table]]+ [:fields {:optional true} [:sequential ::external-field]]])diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 56a212ef83a..5062fef9770 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -444,7 +444,7 @@{:file (byte-array 0)})))))))(deftest metadata-export-basic-test- (testing "GET /api/ee/serialization/metadata/export — happy path with one db/table/field"+ (testing "GET /api/ee/serialization/export/json — happy path with one db/table/field"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"@@ -454,23 +454,23 @@:database_type "BIGINT":semantic_type :type/PK}](let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-databases true:with-tables true:with-fields true)- test-db (m/find-first (comp #{db-name} :id) databases)- test-table (m/find-first (comp #{[db-name "PUBLIC" t-name]} :id) tables)- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]- (is (=? {:id db-name :name db-name :engine "h2"}+ test-db (m/find-first (comp #{db-name} :name) databases)+ test-table (m/find-first #(and (= (:db_id %) db-name) (= (:name %) t-name)) tables)+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])+ (= (:name %) f-name))+ fields)]+ (is (=? {:name db-name :engine "h2"}test-db))- (is (=? {:id [db-name "PUBLIC" t-name]- :db_id db-name+ (is (=? {:db_id db-name:name t-name:schema "PUBLIC":description "A test table"}test-table))- (is (=? {:id [db-name "PUBLIC" t-name f-name]- :table_id [db-name "PUBLIC" t-name]+ (is (=? {:table_id [db-name "PUBLIC" t-name]:name f-name:base_type "type/Integer":database_type "BIGINT"@@ -478,7 +478,7 @@test-field)))))))(deftest metadata-export-optional-properties-test- (testing "GET /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"+ (testing "GET /api/ee/serialization/export/json — effective_type, coercion_strategy and description are emitted when set"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}@@ -489,11 +489,12 @@:coercion_strategy :Coercion/ISO8601->DateTime:description "When this happened"}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-fields true)- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name f-name]} :id) fields)]- (is (=? {:id [db-name "PUBLIC" t-name f-name]- :name f-name+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])+ (= (:name %) f-name))+ fields)]+ (is (=? {:name f-name:base_type "type/Text":effective_type "type/DateTime":coercion_strategy "Coercion/ISO8601->DateTime"@@ -501,28 +502,35 @@test-field)))))))(deftest metadata-export-parent-field-test- (testing "GET /api/ee/serialization/metadata/export — parent_id is nfc_path with the last element dropped, omitted for root fields"+ (testing "GET /api/ee/serialization/export/json — parent_id is emitted only when raw field.parent_id is set"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}:model/Field {root-name :name} {:table_id t-id:base_type :type/Text}- :model/Field _ {:table_id t-id+ :model/Field {parent-id :id} {:table_id t-id+ :name "data"+ :base_type :type/Dictionary}+ :model/Field _ {:table_id t-id+ :name "city":base_type :type/Text+ :parent_id parent-id:nfc_path ["data" "city"]}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ (let [table-id [db-name "PUBLIC" t-name]+ {:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/export/json":with-fields true)- test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name root-name]} :id) fields)- test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name "data" "city"]} :id) fields)]- (is (=? {:id [db-name "PUBLIC" t-name root-name]} test-field-1))+ test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) root-name)) fields)+ test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) "city")) fields)]+ (is (=? {:name root-name :table_id table-id} test-field-1))(is (not (contains? test-field-1 :parent_id)))- (is (=? {:id [db-name "PUBLIC" t-name "data" "city"]+ (is (=? {:name "city"+ :table_id table-id:parent_id [db-name "PUBLIC" t-name "data"]}test-field-2)))))))(deftest metadata-export-fk-test- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is emitted in portable form"+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is emitted in portable form"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}@@ -533,21 +541,22 @@:base_type :type/Integer:semantic_type :type/FK:fk_target_field_id pk-id}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ (let [table-id [db-name "PUBLIC" t-name]+ {:keys [fields]} (mt/user-http-request :crowberto :get 202+ "ee/serialization/export/json":with-fields true)- test-field-1 (m/find-first (comp #{[db-name "PUBLIC" t-name pk-name]} :id) fields)- test-field-2 (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]- (is (=? {:id [db-name "PUBLIC" t-name pk-name] :semantic_type "type/PK"}+ test-field-1 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) pk-name)) fields)+ test-field-2 (m/find-first #(and (= (:table_id %) table-id) (= (:name %) fk-name)) fields)]+ (is (=? {:name pk-name :table_id table-id :semantic_type "type/PK"}test-field-1))- (is (=? {:id [db-name "PUBLIC" t-name fk-name]- :table_id [db-name "PUBLIC" t-name]+ (is (=? {:name fk-name+ :table_id table-id:semantic_type "type/FK"- :fk_target_field_id (:id test-field-1)}+ :fk_target_field_id [db-name "PUBLIC" t-name pk-name]}test-field-2)))))))(deftest metadata-export-hidden-table-test- (testing "GET /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"+ (testing "GET /api/ee/serialization/export/json — hidden tables and their fields are excluded"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id} {:engine :h2}:model/Table {visible-id :id visible-t-name :name} {:db_id db-id :schema "PUBLIC"}@@ -558,7 +567,7 @@:model/Field {hidden-f-name :name} {:table_id hidden-id:base_type :type/Integer}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-tables true :with-fields true)](is (some (comp #{visible-t-name} :name) tables))(is (not-any? (comp #{hidden-t-name} :name) tables))@@ -566,7 +575,7 @@(is (not-any? (comp #{hidden-f-name} :name) fields)))))))(deftest metadata-export-hidden-table-fk-test- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is hidden"+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is hidden"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {visible-id :id v-name :name} {:db_id db-id :schema "PUBLIC"}@@ -580,14 +589,16 @@:semantic_type :type/FK:fk_target_field_id target-id}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-fields true)- test-field (m/find-first (comp #{[db-name "PUBLIC" v-name fk-name]} :id) fields)]+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" v-name])+ (= (:name %) fk-name))+ fields)](is (some? test-field))(is (not (contains? test-field :fk_target_field_id))))))))(deftest metadata-export-inactive-table-test- (testing "GET /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"+ (testing "GET /api/ee/serialization/export/json — inactive tables and all of their fields are excluded"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"@@ -598,14 +609,14 @@:base_type :type/Integer:active false}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-tables true :with-fields true)](is (not-any? (comp #{t-name} :name) tables))(is (not-any? (comp #{active-f-name} :name) fields))(is (not-any? (comp #{inactive-f-name} :name) fields)))))))(deftest metadata-export-sensitive-field-test- (testing "GET /api/ee/serialization/metadata/export — sensitive fields are excluded"+ (testing "GET /api/ee/serialization/export/json — sensitive fields are excluded"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id} {:engine :h2}:model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}@@ -613,13 +624,13 @@:model/Field {secret-name :name} {:table_id t-id :base_type :type/Text:visibility_type :sensitive}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-fields true)](is (some (comp #{email-name} :name) fields))(is (not-any? (comp #{secret-name} :name) fields)))))))(deftest metadata-export-sensitive-fk-test- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target field is sensitive"+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target field is sensitive"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}@@ -631,14 +642,16 @@:semantic_type :type/FK:fk_target_field_id target-id}](let [{:keys [fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-fields true)- test-field (m/find-first (comp #{[db-name "PUBLIC" t-name fk-name]} :id) fields)]+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" t-name])+ (= (:name %) fk-name))+ fields)](is (some? test-field))(is (not (contains? test-field :fk_target_field_id))))))))(deftest metadata-export-inactive-field-test- (testing "GET /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"+ (testing "GET /api/ee/serialization/export/json — inactive fields are excluded; their active sibling and table still appear"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id} {:engine :h2}:model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"}@@ -648,27 +661,27 @@:base_type :type/Integer:active false}](let [{:keys [tables fields]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-tables true :with-fields true)](is (some (comp #{t-name} :name) tables))(is (some (comp #{active-f-name} :name) fields))(is (not-any? (comp #{inactive-f-name} :name) fields)))))))(deftest metadata-export-db-routing-test- (testing "GET /api/ee/serialization/metadata/export — router (mirror) databases are excluded"+ (testing "GET /api/ee/serialization/export/json — router (mirror) databases are excluded"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database _ {:engine :h2}:model/Database {primary-id :id primary-name :name} {:engine :h2}:model/Database {mirror-name :name} {:engine :h2:router_database_id primary-id}](let [{:keys [databases]} (mt/user-http-request :crowberto :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-databases true)](is (some (comp #{primary-name} :name) databases))(is (not-any? (comp #{mirror-name} :name) databases)))))))(deftest metadata-export-table-permission-test- (testing "GET /api/ee/serialization/metadata/export — tables the current user can't access are excluded"+ (testing "GET /api/ee/serialization/export/json — tables the current user can't access are excluded"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id} {:engine :h2}:model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}@@ -685,7 +698,7 @@(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)(let [{:keys [tables fields]} (mt/user-http-request :rasta :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-tables true :with-fields true)](is (some (comp #{accessible-name} :name) tables))(is (not-any? (comp #{restricted-name} :name) tables))@@ -693,7 +706,7 @@(is (not-any? (comp #{no-f-name} :name) fields)))))))(deftest metadata-export-table-permission-fk-test- (testing "GET /api/ee/serialization/metadata/export — fk_target_field_id is dropped when the target table is not accessible"+ (testing "GET /api/ee/serialization/export/json — fk_target_field_id is dropped when the target table is not accessible"(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}:model/Table {accessible-id :id accessible-name :name} {:db_id db-id :schema "PUBLIC"}@@ -713,18 +726,20 @@(perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)(perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)(let [{:keys [fields]} (mt/user-http-request :rasta :get 202- "ee/serialization/metadata/export"+ "ee/serialization/export/json":with-fields true)- test-field (m/find-first (comp #{[db-name "PUBLIC" accessible-name fk-name]} :id) fields)]+ test-field (m/find-first #(and (= (:table_id %) [db-name "PUBLIC" accessible-name])+ (= (:name %) fk-name))+ fields)](is (some? test-field))(is (not (contains? test-field :fk_target_field_id))))))))(deftest metadata-export-token-feature-test- (testing "GET /api/ee/serialization/metadata/export requires the :serialization premium feature"+ (testing "GET /api/ee/serialization/export/json requires the :serialization premium feature"(mt/with-premium-features #{}(mt/assert-has-premium-feature-error"Serialization"- (mt/user-http-request :crowberto :get 402 "ee/serialization/metadata/export")))))+ (mt/user-http-request :crowberto :get 402 "ee/serialization/export/json")))))(deftest serialization-cleanup-test(testing "No temp files are left behind after export/import operations"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.cljnew file mode 100644index 00000000000..df4ae776485--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/export/format_test.clj@@ -0,0 +1,156 @@+(ns metabase-enterprise.serialization.export.format-test+ "Unit tests for `format-entity` in+ `metabase-enterprise.serialization.export.format`. Each deftest exercises a+ single case so that failures pinpoint the exact behavior that broke."+ (:require+ [clojure.test :refer [deftest is]]+ [metabase-enterprise.serialization.export.format :as export.format]))++(defn- mock-database+ "Default :model/Database query row, with per-test overrides merged in."+ [overrides]+ (merge {:name "Sample" :engine "h2"} overrides))++(defn- mock-table+ "Default :model/Table query row, with per-test overrides merged in."+ [overrides]+ (merge {:db_name "Sample"+ :schema "PUBLIC"+ :table_name "Orders"+ :description nil}+ overrides))++(defn- mock-field+ "Default :model/Field query row, with per-test overrides merged in. Covers+ every key destructured by `format-entity :model/Field`, so tests only need to+ set the fields relevant to what they're checking."+ [overrides]+ (merge {:db_name "Sample"+ :engine "h2"+ :table_schema "PUBLIC"+ :table_name "Orders"+ :field_name "id"+ :parent_id nil+ :description nil+ :base_type "type/Integer"+ :database_type nil+ :effective_type nil+ :semantic_type nil+ :coercion_strategy nil+ :nfc_path nil+ :fk_db_name nil+ :fk_db_engine nil+ :fk_table_schema nil+ :fk_table_name nil+ :fk_field_name nil+ :fk_parent_id nil+ :fk_field_nfc_path nil}+ overrides))++(deftest format-database-test+ (is (=? {:name "Sample" :engine "h2"}+ (export.format/format-entity :model/Database (mock-database {})))))++(deftest format-table-with-schema-and-description-test+ (is (=? {:db_id "Sample"+ :name "Orders"+ :schema "PUBLIC"+ :description "Orders table"}+ (export.format/format-entity :model/Table+ (mock-table {:description "Orders table"})))))++(deftest format-table-without-schema-test+ (let [out (export.format/format-entity :model/Table (mock-table {:schema nil}))]+ (is (=? {:db_id "Sample" :name "Orders"} out))+ (is (not (contains? out :schema)))+ (is (not (contains? out :description)))))++(deftest format-field-root-test+ (let [out (export.format/format-entity :model/Field (mock-field {}))]+ (is (=? {:table_id ["Sample" "PUBLIC" "Orders"]+ :name "id"+ :base_type "type/Integer"}+ out))+ (is (not (contains? out :parent_id)))+ (is (not (contains? out :fk_target_field_id)))))++(deftest format-field-mongo-nested-parent-id-test+ (is (=? {:name "city"+ :parent_id ["Sample" "PUBLIC" "Orders" "data"]}+ (export.format/format-entity :model/Field+ (mock-field {:engine "mongo"+ :field_name "city"+ :parent_id 42+ :base_type "type/Text"+ :nfc_path ["data" "city"]})))))++(deftest format-field-bigquery-nested-parent-id-test+ (is (=? {:name "city"+ :parent_id ["Sample" nil "orders" "data"]}+ (export.format/format-entity :model/Field+ (mock-field {:engine "bigquery-cloud-sdk"+ :table_schema nil+ :table_name "orders"+ :field_name "city"+ :parent_id 42+ :base_type "type/Text"+ :nfc_path ["data"]})))))++(deftest format-field-no-parent-id-when-raw-parent-id-missing-test+ (let [out (export.format/format-entity :model/Field+ (mock-field {:engine "mongo"+ :field_name "city"+ :base_type "type/Text"+ :nfc_path ["data" "city"]}))]+ (is (not (contains? out :parent_id)))))++(deftest format-field-fk-target-root-test+ (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "id"]}+ (export.format/format-entity :model/Field+ (mock-field {:field_name "user_id"+ :fk_db_name "Sample"+ :fk_db_engine "h2"+ :fk_table_schema "PUBLIC"+ :fk_table_name "Orders"+ :fk_field_name "id"})))))++(deftest format-field-fk-target-mongo-nested-test+ (is (=? {:fk_target_field_id ["Sample" "PUBLIC" "Orders" "data" "city"]}+ (export.format/format-entity :model/Field+ (mock-field {:engine "mongo"+ :table_name "Customers"+ :field_name "city_ref"+ :base_type "type/Text"+ :fk_db_name "Sample"+ :fk_db_engine "mongo"+ :fk_table_schema "PUBLIC"+ :fk_table_name "Orders"+ :fk_field_name "city"+ :fk_parent_id 99+ :fk_field_nfc_path ["data" "city"]})))))++(deftest format-field-fk-target-bigquery-nested-test+ (is (=? {:fk_target_field_id ["Sample" nil "orders" "data" "city"]}+ (export.format/format-entity :model/Field+ (mock-field {:table_name "Customers"+ :field_name "city_ref"+ :base_type "type/Text"+ :fk_db_name "Sample"+ :fk_db_engine "bigquery-cloud-sdk"+ :fk_table_schema nil+ :fk_table_name "orders"+ :fk_field_name "city"+ :fk_parent_id 99+ :fk_field_nfc_path ["data"]})))))++(deftest format-field-effective-type-redundant-with-base-type-test+ (let [out (export.format/format-entity :model/Field+ (mock-field {:effective_type "type/Integer"}))]+ (is (not (contains? out :effective_type)))))++(deftest format-field-effective-type-distinct-from-base-type-test+ (is (=? {:effective_type "type/Date"}+ (export.format/format-entity :model/Field+ (mock-field {:field_name "created_at"+ :base_type "type/Text"+ :effective_type "type/Date"})))))