Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

drop metadata import api (#73264)

Aleksandr Lesnenko067352f19fc7
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index c87ab718d51..7a4e49666ef 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -783,389 +783,6 @@
(streaming-response {:content-type "application/json; charset=utf-8"} [os _]
(write-field-values! os)))
-;;; ----------------------------------------- POST /api/database/metadata -----------------------------------------
-
-(def ^:private import-batch-size
- "Row batch size for bulk INSERTs and IN-list UPDATEs in POST /api/database/metadata.
- For an 8-column field row, 2000 × 8 = 16k prepared-statement parameters per
- statement — well under Postgres' 65535 cap and MySQL's default max_allowed_packet."
- 2000)
-
-(defn- engine-name
- "Normalize engine (stored as string or keyword) to a string for lookup."
- [engine]
- (when engine (name engine)))
-
-(defn- build-db-lookup
- "Map of [name engine-string] -> target database id."
- []
- (into {}
- (map (juxt (juxt :name (comp engine-name :engine)) :id))
- (t2/select [:model/Database :id :name :engine])))
-
-(defn- build-target-table-lookup
- "{[target-db-id schema table-name] -> target-table-id} for every active table in the given dbs."
- [target-db-ids]
- (if (empty? target-db-ids)
- {}
- (into {}
- (map (juxt (juxt :db_id :schema :name) :id))
- (t2/select [:model/Table :id :db_id :schema :name]
- :db_id [:in target-db-ids] :active true))))
-
-(defn- build-target-field-pathmap
- "{[target-table-id [name1 ... leaf]] -> target-field-id} for every active field under
- the given target tables. Paths are produced by walking `parent_id` on the target side."
- [target-table-ids]
- (if (empty? target-table-ids)
- {}
- (let [rows (t2/select [:model/Field :id :table_id :parent_id :name]
- :table_id [:in target-table-ids] :active true)
- by-id (into {} (map (juxt :id identity)) rows)
- path (fn path [field]
- (if-some [pid (:parent_id field)]
- (conj (path (get by-id pid)) (:name field))
- [(:name field)]))]
- (into {}
- (map (fn [{:keys [id table_id] :as field}]
- [[table_id (path field)] id]))
- rows))))
-
-(def ^:private max-resolution-depth
- "Safety cap on `parent_id` chain walks (`incoming-field-path`, `resolution-depth`)
- so a malformed payload with cycles or deep chains can't stack-overflow or stall
- the import."
- 100)
-
-(defn- incoming-field-path
- "[name1 ... leaf] path of an incoming field, walking parent_id in the payload.
- Cycle-safe: if the parent chain loops or exceeds `max-resolution-depth` the walk
- terminates at that point, treating the field as effectively rooted there."
- [incoming-by-id field-id]
- (loop [id field-id, path (), seen #{}, depth 0]
- (if (or (contains? seen id) (>= depth max-resolution-depth))
- (vec path)
- (if-some [{:keys [name parent_id]} (get incoming-by-id id)]
- (let [path' (conj path name)]
- (if parent_id
- (recur parent_id path' (conj seen id) (inc depth))
- (vec path')))
- (vec path)))))
-
-(defn- resolution-depth
- "Minimum insert-wave at which a new field can be written. Depth 0 means the
- field's parent is already resolvable on the target — nil parent, matched parent
- (already in `in-fld->target`), or orphan parent (payload references a parent that
- isn't in the `fields[]` array). Depth N means the parent is itself a new field
- that must be inserted at wave N-1 first. Cycles and chains deeper than
- `max-resolution-depth` fall back to 0 (root-level insert)."
- [incoming-by-id in-fld->target field-id]
- (loop [id field-id, depth 0, seen #{}]
- (if (or (>= depth max-resolution-depth) (contains? seen id))
- 0
- (let [{:keys [parent_id]} (get incoming-by-id id)]
- (cond
- (nil? parent_id) depth
- (contains? in-fld->target parent_id) depth
- (nil? (get incoming-by-id parent_id)) depth
- :else (recur parent_id (inc depth) (conj seen id)))))))
-
-(defn- new-table-row
- [target-db-id {:keys [schema name description]}]
- (cond-> {:db_id target-db-id
- :name name
- :schema schema
- :active true
- :initial_sync_status "complete"}
- (some? description) (assoc :description description)))
-
-(defn- new-field-row
- "Row for inserting a new field. `parent_id` is resolved by `import-fields!` at
- INSERT time (see that function's docstring for why); `fk_target_field_id` is
- resolved in a later pass after every field exists."
- [target-tbl-id
- {:keys [name base_type description database_type effective_type semantic_type coercion_strategy]}]
- (cond-> {:table_id target-tbl-id
- :name name
- :base_type base_type
- :database_type (or database_type "NULL")
- :active true}
- (some? description) (assoc :description description)
- (some? effective_type) (assoc :effective_type effective_type)
- (some? semantic_type) (assoc :semantic_type semantic_type)
- (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)))
-
-(defn- matched-field-patch
- "Writable keys for a matched field — excludes parent_id (never re-parent matched rows)
- and fk_target_field_id (handled in the references pass)."
- [{:keys [description semantic_type coercion_strategy effective_type]}]
- (cond-> {}
- (some? description) (assoc :description description)
- (some? semantic_type) (assoc :semantic_type semantic_type)
- (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)
- (some? effective_type) (assoc :effective_type effective_type)))
-
-(defn- batched-insert-returning-pks!
- "INSERT `rows` in chunks of `import-batch-size`, returning a vector of pks in
- input order. `into []` drives the transducer to completion, so every chunk's
- `t2/insert-returning-pks!` — and its side effects — runs before this returns."
- [model rows]
- (into []
- (mapcat (fn [chunk] (t2/insert-returning-pks! model chunk)))
- (partition-all import-batch-size rows)))
-
-(defn- import-tables!
- "Phase 1 (per-DB). Match tables by (schema, name) against `target-lookup` (a
- `{[target-db-id schema name] -> target-table-id}` map already scoped to this DB);
- UPDATE matched rows' description per-row, chunked-INSERT the rest via
- `batched-insert-returning-pks!`."
- [state tables target-db-id target-lookup]
- (let [to-insert (volatile! [])]
- (doseq [{:keys [id schema name description] :as tbl} tables]
- (if-some [existing-id (target-lookup [target-db-id schema name])]
- (do
- (when (some? description)
- (t2/update! :model/Table existing-id {:description description}))
- (swap! state #(-> %
- (update-in [:tables :matched] inc)
- (assoc-in [:in-tbl->target id] existing-id))))
- (vswap! to-insert conj [id (new-table-row target-db-id tbl)])))
- (when-some [rows (seq @to-insert)]
- (let [new-ids (batched-insert-returning-pks! :model/Table (mapv second rows))]
- (swap! state (fn [s]
- (reduce (fn [s [[incoming-id _] new-id]]
- (-> s
- (update-in [:tables :created] inc)
- (assoc-in [:in-tbl->target incoming-id] new-id)))
- s
- (map vector rows new-ids))))))))
-
-(defn- import-fields!
- "Phase 2 (per-DB). Classify each incoming field as matched / missing / new, then
- insert new fields in depth-ordered waves so every child's parent_id is resolved
- at INSERT time. Matched rows get per-row UPDATE of editable metadata; parent_id
- on matched rows is never changed and fk_target_field_id is handled by
- `resolve-fk-references!`.
-
- Waves are required — not just convenient — because the `idx_unique_field` DB
- constraint is on `(name, table_id, COALESCE(parent_id, 0))`. Inserting children
- with `parent_id = NULL` would make every root-level row (and every child of a
- not-yet-inserted parent) share `unique_field_helper = 0`, so sibling nested
- leaves that share a leaf name collide. Writing `parent_id` at INSERT time keeps
- each such row under a distinct helper key."
- [state fields incoming-by-id in-tbl->target path-lookup]
- (let [to-insert (volatile! [])]
- (doseq [{:keys [id table_id] :as fld} fields]
- (let [target-tbl (in-tbl->target table_id)
- path (when target-tbl (incoming-field-path incoming-by-id id))
- existing-id (when target-tbl (path-lookup [target-tbl path]))]
- (cond
- (nil? target-tbl)
- (swap! state update-in [:fields :missing] conj
- {:table_id table_id
- :path (incoming-field-path incoming-by-id id)})
-
- existing-id
- (let [patch (matched-field-patch fld)]
- (when (seq patch)
- (t2/update! :model/Field existing-id patch))
- (swap! state #(-> %
- (update-in [:fields :matched] inc)
- (assoc-in [:in-fld->target id] existing-id))))
-
- :else
- (vswap! to-insert conj [id fld target-tbl]))))
- (let [matched-map (:in-fld->target @state)
- by-depth (reduce (fn [acc [id :as triple]]
- (let [depth (resolution-depth incoming-by-id matched-map id)]
- (update acc depth (fnil conj []) triple)))
- (sorted-map)
- @to-insert)]
- (vreset! to-insert nil)
- (doseq [[_depth entries] by-depth]
- (let [in-fld->target (:in-fld->target @state)
- rows (mapv (fn [[_id fld target-tbl]]
- (let [base (new-field-row target-tbl fld)
- pid (get in-fld->target (:parent_id fld))]
- (cond-> base
- pid (assoc :parent_id pid))))
- entries)
- new-ids (batched-insert-returning-pks! :model/Field rows)]
- (swap! state (fn [s]
- (reduce (fn [s [[incoming-id _ _] new-id]]
- (-> s
- (update-in [:fields :created] inc)
- (assoc-in [:in-fld->target incoming-id] new-id)))
- s
- (map vector entries new-ids)))))))))
-
-(defn- classify-fk
- "Classify a field's `fk_target_field_id` against the incoming payload:
- `:same-db` when the target resolves to the same incoming DB; `:cross-db` when
- it resolves to a different DB; `nil` when the field has no FK, or the target
- isn't in the payload, or its DB can't be determined."
- [incoming-by-id table->db {:keys [fk_target_field_id table_id]}]
- (when-some [target (get incoming-by-id fk_target_field_id)]
- (let [src (table->db table_id)
- dst (table->db (:table_id target))]
- (cond
- (nil? dst) nil
- (= src dst) :same-db
- :else :cross-db))))
-
-(defn- fk-pairs-of-kind
- "Return `[[self-tid fk-tid] ...]` for every field in `fields` whose fk classifies
- as `kind` and whose self-id AND fk target both resolved in `in-fld->target`."
- [fields incoming-by-id table->db in-fld->target kind]
- (into []
- (keep (fn [{:keys [id fk_target_field_id] :as fld}]
- (when (= kind (classify-fk incoming-by-id table->db fld))
- (let [self-tid (in-fld->target id)
- fk-tid (in-fld->target fk_target_field_id)]
- (when (and self-tid fk-tid) [self-tid fk-tid])))))
- fields))
-
-(defn- resolve-fk-references!
- "Phase 3. Write `fk_target_field_id` for each `[self-tid fk-tid]` pair, grouping
- by `fk-tid` so many sources pointing at the same target collapse into a single
- `UPDATE ... WHERE id IN (...)`, chunked to `import-batch-size` ids per statement.
- `parent_id` is NOT handled here — it's written at INSERT time by `import-fields!`."
- [pairs]
- (let [buckets (reduce (fn [acc [self-tid fk-tid]]
- (update acc fk-tid (fnil conj []) self-tid))
- {}
- pairs)]
- (doseq [[fk-tid ids] buckets
- id-chunk (partition-all import-batch-size ids)]
- (t2/update! :model/Field :id [:in id-chunk] {:fk_target_field_id fk-tid}))))
-
-(defn- import-metadata!*
- "Core logic for POST /metadata. See the endpoint doc for behavior.
-
- Runs a separate `t2/with-transaction` per matched target DB: each DB's tables,
- fields, and same-DB FK references are imported in isolation so one DB's failure
- does not roll back the rest, and memory usage is bounded by the largest single
- DB's target-field-pathmap rather than the whole payload. After every per-DB
- transaction has committed, a final transaction resolves `fk_target_field_id`
- references that cross DB boundaries."
- [{:keys [databases tables fields]}]
- (let [db-by-key (build-db-lookup)
- in-db->target (into {}
- (keep (fn [{:keys [id name engine]}]
- (when-some [tid (db-by-key [name (engine-name engine)])]
- [id tid])))
- databases)
- missing-dbs (mapv #(select-keys % [:name :engine])
- (remove #(in-db->target (:id %)) databases))
- incoming-by-id (m/index-by :id fields)
- table->db (into {} (map (juxt :id :db_id)) tables)
- tables-by-db (group-by :db_id tables)
- fields-by-db (group-by #(table->db (:table_id %)) fields)
- ;; Tables/fields whose DB isn't matched, or fields whose table_id isn't in
- ;; tables[], can't be placed — record as missing up front so per-DB loops
- ;; only handle processable rows.
- missing-tables (into []
- (keep (fn [{:keys [db_id schema name]}]
- (when-not (in-db->target db_id)
- (cond-> {:db_id db_id :name name}
- (some? schema) (assoc :schema schema)))))
- tables)
- missing-fields (into []
- (keep (fn [{:keys [id table_id]}]
- (let [incoming-db (table->db table_id)]
- (when (or (nil? incoming-db)
- (not (in-db->target incoming-db)))
- {:table_id table_id
- :path (incoming-field-path incoming-by-id id)}))))
- fields)
- global-state (atom {:tables {:matched 0 :created 0 :missing missing-tables}
- :fields {:matched 0 :created 0 :missing missing-fields}
- :in-fld->target {}
- :failed-dbs []})]
- (doseq [[incoming-db-id target-db-id] in-db->target]
- (let [db-tables (get tables-by-db incoming-db-id [])
- db-fields (get fields-by-db incoming-db-id [])
- db-state (atom {:tables (:tables @global-state)
- :fields (:fields @global-state)
- :in-tbl->target {}
- :in-fld->target (:in-fld->target @global-state)})]
- (try
- (t2/with-transaction [_conn]
- (let [target-tbl-lookup (build-target-table-lookup [target-db-id])]
- (import-tables! db-state db-tables target-db-id target-tbl-lookup))
- (let [in-tbl->target (:in-tbl->target @db-state)
- path-lookup (build-target-field-pathmap (vals in-tbl->target))]
- (import-fields! db-state db-fields incoming-by-id in-tbl->target path-lookup))
- (resolve-fk-references!
- (fk-pairs-of-kind db-fields incoming-by-id table->db
- (:in-fld->target @db-state) :same-db)))
- (swap! global-state
- (fn [g]
- (-> g
- (assoc :tables (:tables @db-state))
- (assoc :fields (:fields @db-state))
- (update :in-fld->target merge (:in-fld->target @db-state)))))
- (catch Throwable t
- (log/errorf t "POST /api/database/metadata: import failed for db %s (target %s)"
- incoming-db-id target-db-id)
- (swap! global-state update :failed-dbs conj
- {:id incoming-db-id :target target-db-id :error (.getMessage t)})))))
- (let [cross-db-pairs (fk-pairs-of-kind fields incoming-by-id table->db
- (:in-fld->target @global-state) :cross-db)]
- (when (seq cross-db-pairs)
- (t2/with-transaction [_conn]
- (resolve-fk-references! cross-db-pairs))))
- (let [{tbl-report :tables fld-report :fields failed-dbs :failed-dbs} @global-state]
- {:databases (cond-> {:matched (count in-db->target)
- :missing missing-dbs}
- (seq failed-dbs) (assoc :failed failed-dbs))
- :tables tbl-report
- :fields fld-report})))
-
-(mr/def ::metadata-import-report
- [:map
- [:databases [:map
- [:matched :int]
- [:missing [:sequential :map]]
- [:failed {:optional true} [:sequential :map]]]]
- [:tables [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]
- [:fields [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]])
-
-(api.macros/defendpoint :post "/metadata"
- :- ::metadata-import-report
- "Import database/table/field metadata previously exported from `GET /api/database/metadata`.
-
- Entities are matched by natural key — databases by `(name, engine)`, tables by
- `(database, schema, name)`, fields by `(table, parent-path, name)` — so the numeric ids
- in the payload are only used to link fields to their tables within the request.
-
- Tables and fields that don't exist on the target are created when their parent
- (database for tables; table for fields) is present on the target. Databases are
- not auto-created — missing databases are reported instead. Field `fk_target_field_id`
- is re-resolved through the natural-key lookup after all fields exist.
-
- For matched entities, only user-editable metadata is written: table `description`;
- field `description`, `semantic_type`, `coercion_strategy`, `effective_type`, and
- `fk_target_field_id`. For newly-created fields, `base_type` and `database_type` are
- also populated from the payload. Keys absent from the payload (including null values)
- are left untouched on matched entities.
-
- Processing is isolated per target database: each matched DB imports in its own
- transaction so a failure on one DB does not roll back the others. DBs whose
- transaction failed appear under `databases.failed` in the response along with
- the error message; every other DB's tables, fields, and same-DB `fk_target_field_id`
- references still commit. Cross-database `fk_target_field_id` references are
- resolved in a final pass after all per-DB transactions have committed.
-
- Returns counts of matched + created entities per type, a list of entities in the
- payload that could not be placed (their parent was missing on the target), and,
- when any DB failed, a `databases.failed` list naming each failed DB."
- [_route-params
- _query-params
- body :- ::databases-metadata-response]
- (api/check-superuser)
- (import-metadata!* body))
-
;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------
;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbers
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index a8af32d5b74..c80ac3e959b 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -874,360 +874,6 @@
(is (nil? (m/find-first (comp #{t-id} :id) tables)))
(is (nil? (m/find-first (comp #{f-id} :id) fields)))))))
-(deftest databases-metadata-import-test
- (testing "POST /api/database/metadata"
- (mt/with-temp [:model/Database {db-id :id} {:name "import-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "orders" :schema "PUBLIC"
- :description "original"}
- :model/Field {pk-id :id} {:table_id t-id :name "id" :base_type :type/Integer
- :database_type "BIGINT"}
- :model/Field {fk-id :id} {:table_id t-id :name "order_id" :base_type :type/Integer
- :database_type "BIGINT"}]
- (testing "matched entities are updated; missing tables/fields are created when parent exists"
- ;; Payload carries ids from "another" instance — here we reuse our own ids, but the
- ;; endpoint matches by natural key regardless of what the numeric ids are.
- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}
- {:id 9999 :name "does-not-exist" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"
- :description "updated via import"}
- {:id 9998 :db_id db-id :name "new_table" :schema "PUBLIC"
- :description "created via import"}]
- :fields [{:id pk-id :table_id t-id :name "id"
- :base_type "type/Integer" :database_type "BIGINT"
- :semantic_type "type/PK"
- :description "primary key"}
- {:id fk-id :table_id t-id :name "order_id"
- :base_type "type/Integer" :database_type "BIGINT"
- :semantic_type "type/FK"
- :fk_target_field_id pk-id}
- {:id 9997 :table_id t-id :name "new_field"
- :base_type "type/Integer" :database_type "INT"
- :description "created via import"
- :semantic_type "type/Quantity"}
- {:id 9996 :table_id 9998 :name "new_table_field"
- :base_type "type/Text" :database_type "VARCHAR"}]}
- report (mt/user-http-request :crowberto :post 200
- "database/metadata" payload)]
- (is (=? {:databases {:matched 1 :missing [{:name "does-not-exist"}]}
- :tables {:matched 1 :created 1 :missing []}
- :fields {:matched 2 :created 2 :missing []}}
- report))
- (is (= "updated via import" (t2/select-one-fn :description :model/Table :id t-id)))
- (is (= :type/PK (t2/select-one-fn :semantic_type :model/Field :id pk-id)))
- (is (= "primary key" (t2/select-one-fn :description :model/Field :id pk-id)))
- (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))
- (testing "new table was created under the matched database"
- (let [new-tbl (t2/select-one :model/Table :db_id db-id :name "new_table")]
- (is (some? new-tbl))
- (is (= "created via import" (:description new-tbl)))
- (is (true? (:active new-tbl)))))
- (testing "new field was created under the existing table"
- (let [new-fld (t2/select-one :model/Field :table_id t-id :name "new_field")]
- (is (some? new-fld))
- (is (= :type/Integer (:base_type new-fld)))
- (is (= "INT" (:database_type new-fld)))
- (is (= "created via import" (:description new-fld)))
- (is (= :type/Quantity (:semantic_type new-fld)))))
- (testing "new field was created under a newly-created table"
- (let [new-tbl-id (t2/select-one-pk :model/Table :db_id db-id :name "new_table")]
- (is (some? (t2/select-one :model/Field :table_id new-tbl-id :name "new_table_field")))))))
-
- (testing "fields whose database is missing on the target are reported as missing"
- (let [payload {:databases [{:id 9999 :name "does-not-exist" :engine "h2"}]
- :tables [{:id 9998 :db_id 9999 :name "x" :schema "PUBLIC"}]
- :fields [{:id 9997 :table_id 9998 :name "y"
- :base_type "type/Integer" :database_type "INTEGER"}]}
- report (mt/user-http-request :crowberto :post 200
- "database/metadata" payload)]
- (is (=? {:tables {:matched 0 :created 0 :missing [{:name "x"}]}
- :fields {:matched 0 :created 0 :missing [{:path ["y"]}]}}
- report))))
-
- (testing "base_type and database_type are never overwritten"
- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]
- :fields [{:id pk-id :table_id t-id :name "id"
- :base_type "type/Text" :database_type "TEXT"
- :description "still a pk"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (is (= :type/Integer (t2/select-one-fn :base_type :model/Field :id pk-id)))
- (is (= "BIGINT" (t2/select-one-fn :database_type :model/Field :id pk-id)))))
-
- (testing "nested fields are matched by parent path"
- (mt/with-temp [:model/Field {parent-id :id} {:table_id t-id :name "payload"
- :base_type :type/JSON :database_type "JSON"}
- :model/Field {child-id :id} {:table_id t-id :parent_id parent-id :name "amount"
- :base_type :type/Integer :database_type "BIGINT"}]
- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-id :name "payload"
- :base_type "type/JSON" :database_type "JSON"}
- {:id 2 :table_id t-id :parent_id 1 :name "amount"
- :base_type "type/Integer" :database_type "BIGINT"
- :description "nested description"
- :semantic_type "type/Quantity"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (is (= "nested description" (t2/select-one-fn :description :model/Field :id child-id)))
- (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id child-id))))))
-
- (testing "fields with the same leaf name at different parent paths are matched independently"
- ;; A root-level `amount` and a nested `payload.amount` coexist on the same table.
- ;; Matching by full parent path must update each without clobbering the other.
- (mt/with-temp [:model/Field {root-amount-id :id} {:table_id t-id :name "amount"
- :base_type :type/Integer :database_type "BIGINT"}
- :model/Field {parent-id :id} {:table_id t-id :name "payload"
- :base_type :type/JSON :database_type "JSON"}
- :model/Field {nested-amount-id :id} {:table_id t-id :parent_id parent-id :name "amount"
- :base_type :type/Integer :database_type "BIGINT"}]
- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]
- :fields [{:id 10 :table_id t-id :name "amount"
- :base_type "type/Integer" :database_type "BIGINT"
- :description "root amount"
- :semantic_type "type/Quantity"}
- {:id 11 :table_id t-id :name "payload"
- :base_type "type/JSON" :database_type "JSON"}
- {:id 12 :table_id t-id :parent_id 11 :name "amount"
- :base_type "type/Integer" :database_type "BIGINT"
- :description "nested amount"
- :semantic_type "type/Currency"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (is (= "root amount" (t2/select-one-fn :description :model/Field :id root-amount-id)))
- (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id root-amount-id)))
- (is (= "nested amount" (t2/select-one-fn :description :model/Field :id nested-amount-id)))
- (is (= :type/Currency (t2/select-one-fn :semantic_type :model/Field :id nested-amount-id))))))
-
- (testing "non-superusers are rejected"
- (mt/user-http-request :rasta :post 403 "database/metadata"
- {:databases [] :tables [] :fields []})))))
-
-(deftest databases-metadata-import-nested-fields-test
- (testing "POST /api/database/metadata — nested fields with shared leaf names under siblings"
- (mt/with-temp [:model/Database {db-id :id} {:name "nested-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "nested-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]
- :fields [{:id 100 :table_id t-id :name "wind"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 101 :table_id t-id :parent_id 100 :name "value"
- :base_type "type/Float" :database_type "NULL"}
- {:id 102 :table_id t-id :name "temp"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 103 :table_id t-id :parent_id 102 :name "value"
- :base_type "type/Float" :database_type "NULL"}]}
- report (mt/user-http-request :crowberto :post 200
- "database/metadata" payload)
- by-name (->> (t2/select [:model/Field :id :name :parent_id] :table_id t-id)
- (group-by :name))
- wind-id (:id (first (get by-name "wind")))
- temp-id (:id (first (get by-name "temp")))
- values (sort-by :parent_id (get by-name "value"))]
- (is (=? {:databases {:matched 1 :missing []}
- :tables {:matched 1 :created 0 :missing []}
- :fields {:matched 0 :created 4 :missing []}}
- report))
- (is (= 4 (t2/count :model/Field :table_id t-id)))
- (is (= 2 (count values)))
- (testing "each value row points at its own parent, not NULL"
- (is (= #{wind-id temp-id}
- (set (map :parent_id values))))))))
-
- (testing "POST /api/database/metadata — 3-level deep nesting with correct parent chain"
- (mt/with-temp [:model/Database {db-id :id} {:name "nested-3-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "tree" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "nested-3-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "tree" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-id :name "a"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 2 :table_id t-id :parent_id 1 :name "b"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 3 :table_id t-id :parent_id 2 :name "c"
- :base_type "type/Integer" :database_type "NULL"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (let [a (t2/select-one :model/Field :table_id t-id :name "a")
- b (t2/select-one :model/Field :table_id t-id :name "b")
- c (t2/select-one :model/Field :table_id t-id :name "c")]
- (is (nil? (:parent_id a)))
- (is (= (:id a) (:parent_id b)))
- (is (= (:id b) (:parent_id c)))))))
-
- (testing "POST /api/database/metadata — new child attaches to a pre-existing matched parent"
- (mt/with-temp [:model/Database {db-id :id} {:name "matched-parent-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}
- :model/Field {p-id :id} {:table_id t-id :name "payload"
- :base_type :type/Dictionary}]
- (let [payload {:databases [{:id db-id :name "matched-parent-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]
- :fields [{:id 10 :table_id t-id :name "payload"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 11 :table_id t-id :parent_id 10 :name "new_leaf"
- :base_type "type/Integer" :database_type "NULL"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (let [child (t2/select-one :model/Field :table_id t-id :name "new_leaf")]
- (is (some? child))
- (is (= p-id (:parent_id child)))))))
-
- (testing "POST /api/database/metadata — nested-field import is idempotent on repeat"
- (mt/with-temp [:model/Database {db-id :id} {:name "idempotency-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "idempotency-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]
- :fields [{:id 100 :table_id t-id :name "wind"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 101 :table_id t-id :parent_id 100 :name "value"
- :base_type "type/Float" :database_type "NULL"}
- {:id 102 :table_id t-id :name "temp"
- :base_type "type/Dictionary" :database_type "NULL"}
- {:id 103 :table_id t-id :parent_id 102 :name "value"
- :base_type "type/Float" :database_type "NULL"}]}
- first-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- second-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]
- (is (=? {:fields {:matched 0 :created 4}} first-report))
- (is (=? {:fields {:matched 4 :created 0}} second-report))
- (is (= 4 (t2/count :model/Field :table_id t-id)))))))
-
-(deftest databases-metadata-import-edge-cases-test
- (testing "POST /api/database/metadata — orphan parent_id (references a field not in payload) lands as root"
- (mt/with-temp [:model/Database {db-id :id} {:name "orphan-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "orphan-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]
- :fields [{:id 99 :table_id t-id :parent_id 9999 :name "orphan"
- :base_type "type/Text" :database_type "TEXT"}]}
- report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]
- (is (=? {:fields {:created 1}} report))
- (let [orphan (t2/select-one :model/Field :table_id t-id :name "orphan")]
- (is (some? orphan))
- (is (nil? (:parent_id orphan)))))))
-
- (testing "POST /api/database/metadata — omitted database_type falls back to the \"NULL\" sentinel"
- ;; GET's `format-field-metadata` drops nil `database_type` via `m/assoc-some`,
- ;; so a round-tripped payload from a MySQL app DB (nullable column) can omit
- ;; the key. The sentinel keeps the INSERT valid under Postgres' NOT NULL.
- (mt/with-temp [:model/Database {db-id :id} {:name "no-db-type-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "no-db-type-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-id :name "mongo_field"
- :base_type "type/Text"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (is (= "NULL"
- (t2/select-one-fn :database_type :model/Field :table_id t-id :name "mongo_field"))))))
-
- (testing "POST /api/database/metadata — cycle in incoming parent_id does not stack-overflow"
- (mt/with-temp [:model/Database {db-id :id} {:name "cycle-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]
- (let [payload {:databases [{:id db-id :name "cycle-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-id :parent_id 2 :name "a"
- :base_type "type/Text" :database_type "TEXT"}
- {:id 2 :table_id t-id :parent_id 1 :name "b"
- :base_type "type/Text" :database_type "TEXT"}]}]
- ;; Degenerate payload — only the status code is asserted; the rows' shape
- ;; after a cycle-broken insert is intentionally unspecified.
- (is (=? {:databases {:matched 1}}
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)))))))
-
-(deftest databases-metadata-import-batching-test
- (testing "POST /api/database/metadata — new-field INSERTs are chunked by import-batch-size"
- (mt/with-temp [:model/Database {db-id :id} {:name "batch-db" :engine :h2}
- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]
- (let [insert-calls (atom [])
- orig-insert t2/insert-returning-pks!
- payload {:databases [{:id db-id :name "batch-db" :engine "h2"}]
- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]
- :fields (mapv (fn [idx]
- {:id (+ 1000 idx)
- :table_id t-id
- :name (str "col_" idx)
- :base_type "type/Integer"
- :database_type "INTEGER"})
- (range 7))}]
- (with-redefs [api.database/import-batch-size 3
- t2/insert-returning-pks! (fn [model rows]
- (when (= model :model/Field)
- (swap! insert-calls conj (count rows)))
- (orig-insert model rows))]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload))
- (is (= [3 3 1] @insert-calls))))))
-
-(deftest databases-metadata-import-partial-failure-test
- (testing "POST /api/database/metadata — one DB's failure does not roll back others"
- (mt/with-temp [:model/Database {db-a-id :id} {:name "pf-db-a" :engine :h2}
- :model/Database {db-b-id :id} {:name "pf-db-b" :engine :h2}
- :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}
- :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}]
- (let [orig-import-fields (deref #'api.database/import-fields!)]
- (with-redefs [api.database/import-fields! (fn [state fields incoming-by-id in-tbl->target path-lookup]
- (if (some #(= t-b-id (:table_id %)) fields)
- (throw (ex-info "injected failure for DB-B" {}))
- (orig-import-fields state fields incoming-by-id
- in-tbl->target path-lookup)))]
- (let [payload {:databases [{:id db-a-id :name "pf-db-a" :engine "h2"}
- {:id db-b-id :name "pf-db-b" :engine "h2"}]
- :tables [{:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}
- {:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-a-id :name "a_col"
- :base_type "type/Integer" :database_type "INTEGER"}
- {:id 2 :table_id t-b-id :name "b_col"
- :base_type "type/Integer" :database_type "INTEGER"}]}
- report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]
- (is (=? {:databases {:matched 2
- :missing []
- :failed [{:id db-b-id :target db-b-id}]}}
- report))
- (testing "DB-A's field committed despite DB-B's failure"
- (is (some? (t2/select-one :model/Field :table_id t-a-id :name "a_col"))))
- (testing "DB-B's field did not commit (transaction rolled back)"
- (is (nil? (t2/select-one :model/Field :table_id t-b-id :name "b_col")))))))))
-
- (testing "POST /api/database/metadata — pathmap is scoped to the current DB only"
- (mt/with-temp [:model/Database {db-a-id :id} {:name "scope-db-a" :engine :h2}
- :model/Database {db-b-id :id} {:name "scope-db-b" :engine :h2}
- :model/Table {t-a-id :id} {:db_id db-a-id :name "t" :schema "PUBLIC"}
- :model/Table {t-b-id :id} {:db_id db-b-id :name "t" :schema "PUBLIC"}]
- (let [calls (atom [])
- orig-build-pm (deref #'api.database/build-target-field-pathmap)]
- (with-redefs [api.database/build-target-field-pathmap (fn [ids]
- (swap! calls conj (set ids))
- (orig-build-pm ids))]
- (let [payload {:databases [{:id db-a-id :name "scope-db-a" :engine "h2"}]
- :tables [{:id t-a-id :db_id db-a-id :name "t" :schema "PUBLIC"}]
- :fields [{:id 1 :table_id t-a-id :name "c"
- :base_type "type/Integer" :database_type "INTEGER"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload))
- (testing "build-target-field-pathmap never saw db-b's table"
- (is (every? #(not (contains? % t-b-id)) @calls)))
- (testing "and was called with db-a's table"
- (is (some #(contains? % t-a-id) @calls))))))))
-
-(deftest databases-metadata-import-cross-db-fk-test
- (testing "POST /api/database/metadata — cross-DB fk_target_field_id resolves after all DBs commit"
- (mt/with-temp [:model/Database {db-a-id :id} {:name "xfk-db-a" :engine :h2}
- :model/Database {db-b-id :id} {:name "xfk-db-b" :engine :h2}
- :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}
- :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}
- :model/Field {pk-id :id} {:table_id t-a-id :name "id"
- :base_type :type/Integer
- :database_type "BIGINT"
- :semantic_type :type/PK}
- :model/Field {fk-id :id} {:table_id t-b-id :name "a_id"
- :base_type :type/Integer
- :database_type "BIGINT"
- :semantic_type :type/FK}]
- (let [payload {:databases [{:id db-b-id :name "xfk-db-b" :engine "h2"}
- {:id db-a-id :name "xfk-db-a" :engine "h2"}]
- :tables [{:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}
- {:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}]
- :fields [{:id fk-id :table_id t-b-id :name "a_id"
- :base_type "type/Integer" :database_type "BIGINT"
- :fk_target_field_id pk-id}
- {:id pk-id :table_id t-a-id :name "id"
- :base_type "type/Integer" :database_type "BIGINT"
- :semantic_type "type/PK"}]}]
- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)
- (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))))))
-
(deftest ^:parallel fetch-database-metadata-test
(testing "GET /api/database/:id/metadata"
(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)