The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
drop metadata import api (#73264)
Aleksandr Lesnenko067352f19fc7
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex c87ab718d51..7a4e49666ef 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -783,389 +783,6 @@(streaming-response {:content-type "application/json; charset=utf-8"} [os _](write-field-values! os)))-;;; ----------------------------------------- POST /api/database/metadata -------------------------------------------(def ^:private import-batch-size- "Row batch size for bulk INSERTs and IN-list UPDATEs in POST /api/database/metadata.- For an 8-column field row, 2000 × 8 = 16k prepared-statement parameters per- statement — well under Postgres' 65535 cap and MySQL's default max_allowed_packet."- 2000)--(defn- engine-name- "Normalize engine (stored as string or keyword) to a string for lookup."- [engine]- (when engine (name engine)))--(defn- build-db-lookup- "Map of [name engine-string] -> target database id."- []- (into {}- (map (juxt (juxt :name (comp engine-name :engine)) :id))- (t2/select [:model/Database :id :name :engine])))--(defn- build-target-table-lookup- "{[target-db-id schema table-name] -> target-table-id} for every active table in the given dbs."- [target-db-ids]- (if (empty? target-db-ids)- {}- (into {}- (map (juxt (juxt :db_id :schema :name) :id))- (t2/select [:model/Table :id :db_id :schema :name]- :db_id [:in target-db-ids] :active true))))--(defn- build-target-field-pathmap- "{[target-table-id [name1 ... leaf]] -> target-field-id} for every active field under- the given target tables. Paths are produced by walking `parent_id` on the target side."- [target-table-ids]- (if (empty? target-table-ids)- {}- (let [rows (t2/select [:model/Field :id :table_id :parent_id :name]- :table_id [:in target-table-ids] :active true)- by-id (into {} (map (juxt :id identity)) rows)- path (fn path [field]- (if-some [pid (:parent_id field)]- (conj (path (get by-id pid)) (:name field))- [(:name field)]))]- (into {}- (map (fn [{:keys [id table_id] :as field}]- [[table_id (path field)] id]))- rows))))--(def ^:private max-resolution-depth- "Safety cap on `parent_id` chain walks (`incoming-field-path`, `resolution-depth`)- so a malformed payload with cycles or deep chains can't stack-overflow or stall- the import."- 100)--(defn- incoming-field-path- "[name1 ... leaf] path of an incoming field, walking parent_id in the payload.- Cycle-safe: if the parent chain loops or exceeds `max-resolution-depth` the walk- terminates at that point, treating the field as effectively rooted there."- [incoming-by-id field-id]- (loop [id field-id, path (), seen #{}, depth 0]- (if (or (contains? seen id) (>= depth max-resolution-depth))- (vec path)- (if-some [{:keys [name parent_id]} (get incoming-by-id id)]- (let [path' (conj path name)]- (if parent_id- (recur parent_id path' (conj seen id) (inc depth))- (vec path')))- (vec path)))))--(defn- resolution-depth- "Minimum insert-wave at which a new field can be written. Depth 0 means the- field's parent is already resolvable on the target — nil parent, matched parent- (already in `in-fld->target`), or orphan parent (payload references a parent that- isn't in the `fields[]` array). Depth N means the parent is itself a new field- that must be inserted at wave N-1 first. Cycles and chains deeper than- `max-resolution-depth` fall back to 0 (root-level insert)."- [incoming-by-id in-fld->target field-id]- (loop [id field-id, depth 0, seen #{}]- (if (or (>= depth max-resolution-depth) (contains? seen id))- 0- (let [{:keys [parent_id]} (get incoming-by-id id)]- (cond- (nil? parent_id) depth- (contains? in-fld->target parent_id) depth- (nil? (get incoming-by-id parent_id)) depth- :else (recur parent_id (inc depth) (conj seen id)))))))--(defn- new-table-row- [target-db-id {:keys [schema name description]}]- (cond-> {:db_id target-db-id- :name name- :schema schema- :active true- :initial_sync_status "complete"}- (some? description) (assoc :description description)))--(defn- new-field-row- "Row for inserting a new field. `parent_id` is resolved by `import-fields!` at- INSERT time (see that function's docstring for why); `fk_target_field_id` is- resolved in a later pass after every field exists."- [target-tbl-id- {:keys [name base_type description database_type effective_type semantic_type coercion_strategy]}]- (cond-> {:table_id target-tbl-id- :name name- :base_type base_type- :database_type (or database_type "NULL")- :active true}- (some? description) (assoc :description description)- (some? effective_type) (assoc :effective_type effective_type)- (some? semantic_type) (assoc :semantic_type semantic_type)- (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)))--(defn- matched-field-patch- "Writable keys for a matched field — excludes parent_id (never re-parent matched rows)- and fk_target_field_id (handled in the references pass)."- [{:keys [description semantic_type coercion_strategy effective_type]}]- (cond-> {}- (some? description) (assoc :description description)- (some? semantic_type) (assoc :semantic_type semantic_type)- (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)- (some? effective_type) (assoc :effective_type effective_type)))--(defn- batched-insert-returning-pks!- "INSERT `rows` in chunks of `import-batch-size`, returning a vector of pks in- input order. `into []` drives the transducer to completion, so every chunk's- `t2/insert-returning-pks!` — and its side effects — runs before this returns."- [model rows]- (into []- (mapcat (fn [chunk] (t2/insert-returning-pks! model chunk)))- (partition-all import-batch-size rows)))--(defn- import-tables!- "Phase 1 (per-DB). Match tables by (schema, name) against `target-lookup` (a- `{[target-db-id schema name] -> target-table-id}` map already scoped to this DB);- UPDATE matched rows' description per-row, chunked-INSERT the rest via- `batched-insert-returning-pks!`."- [state tables target-db-id target-lookup]- (let [to-insert (volatile! [])]- (doseq [{:keys [id schema name description] :as tbl} tables]- (if-some [existing-id (target-lookup [target-db-id schema name])]- (do- (when (some? description)- (t2/update! :model/Table existing-id {:description description}))- (swap! state #(-> %- (update-in [:tables :matched] inc)- (assoc-in [:in-tbl->target id] existing-id))))- (vswap! to-insert conj [id (new-table-row target-db-id tbl)])))- (when-some [rows (seq @to-insert)]- (let [new-ids (batched-insert-returning-pks! :model/Table (mapv second rows))]- (swap! state (fn [s]- (reduce (fn [s [[incoming-id _] new-id]]- (-> s- (update-in [:tables :created] inc)- (assoc-in [:in-tbl->target incoming-id] new-id)))- s- (map vector rows new-ids))))))))--(defn- import-fields!- "Phase 2 (per-DB). Classify each incoming field as matched / missing / new, then- insert new fields in depth-ordered waves so every child's parent_id is resolved- at INSERT time. Matched rows get per-row UPDATE of editable metadata; parent_id- on matched rows is never changed and fk_target_field_id is handled by- `resolve-fk-references!`.-- Waves are required — not just convenient — because the `idx_unique_field` DB- constraint is on `(name, table_id, COALESCE(parent_id, 0))`. Inserting children- with `parent_id = NULL` would make every root-level row (and every child of a- not-yet-inserted parent) share `unique_field_helper = 0`, so sibling nested- leaves that share a leaf name collide. Writing `parent_id` at INSERT time keeps- each such row under a distinct helper key."- [state fields incoming-by-id in-tbl->target path-lookup]- (let [to-insert (volatile! [])]- (doseq [{:keys [id table_id] :as fld} fields]- (let [target-tbl (in-tbl->target table_id)- path (when target-tbl (incoming-field-path incoming-by-id id))- existing-id (when target-tbl (path-lookup [target-tbl path]))]- (cond- (nil? target-tbl)- (swap! state update-in [:fields :missing] conj- {:table_id table_id- :path (incoming-field-path incoming-by-id id)})-- existing-id- (let [patch (matched-field-patch fld)]- (when (seq patch)- (t2/update! :model/Field existing-id patch))- (swap! state #(-> %- (update-in [:fields :matched] inc)- (assoc-in [:in-fld->target id] existing-id))))-- :else- (vswap! to-insert conj [id fld target-tbl]))))- (let [matched-map (:in-fld->target @state)- by-depth (reduce (fn [acc [id :as triple]]- (let [depth (resolution-depth incoming-by-id matched-map id)]- (update acc depth (fnil conj []) triple)))- (sorted-map)- @to-insert)]- (vreset! to-insert nil)- (doseq [[_depth entries] by-depth]- (let [in-fld->target (:in-fld->target @state)- rows (mapv (fn [[_id fld target-tbl]]- (let [base (new-field-row target-tbl fld)- pid (get in-fld->target (:parent_id fld))]- (cond-> base- pid (assoc :parent_id pid))))- entries)- new-ids (batched-insert-returning-pks! :model/Field rows)]- (swap! state (fn [s]- (reduce (fn [s [[incoming-id _ _] new-id]]- (-> s- (update-in [:fields :created] inc)- (assoc-in [:in-fld->target incoming-id] new-id)))- s- (map vector entries new-ids)))))))))--(defn- classify-fk- "Classify a field's `fk_target_field_id` against the incoming payload:- `:same-db` when the target resolves to the same incoming DB; `:cross-db` when- it resolves to a different DB; `nil` when the field has no FK, or the target- isn't in the payload, or its DB can't be determined."- [incoming-by-id table->db {:keys [fk_target_field_id table_id]}]- (when-some [target (get incoming-by-id fk_target_field_id)]- (let [src (table->db table_id)- dst (table->db (:table_id target))]- (cond- (nil? dst) nil- (= src dst) :same-db- :else :cross-db))))--(defn- fk-pairs-of-kind- "Return `[[self-tid fk-tid] ...]` for every field in `fields` whose fk classifies- as `kind` and whose self-id AND fk target both resolved in `in-fld->target`."- [fields incoming-by-id table->db in-fld->target kind]- (into []- (keep (fn [{:keys [id fk_target_field_id] :as fld}]- (when (= kind (classify-fk incoming-by-id table->db fld))- (let [self-tid (in-fld->target id)- fk-tid (in-fld->target fk_target_field_id)]- (when (and self-tid fk-tid) [self-tid fk-tid])))))- fields))--(defn- resolve-fk-references!- "Phase 3. Write `fk_target_field_id` for each `[self-tid fk-tid]` pair, grouping- by `fk-tid` so many sources pointing at the same target collapse into a single- `UPDATE ... WHERE id IN (...)`, chunked to `import-batch-size` ids per statement.- `parent_id` is NOT handled here — it's written at INSERT time by `import-fields!`."- [pairs]- (let [buckets (reduce (fn [acc [self-tid fk-tid]]- (update acc fk-tid (fnil conj []) self-tid))- {}- pairs)]- (doseq [[fk-tid ids] buckets- id-chunk (partition-all import-batch-size ids)]- (t2/update! :model/Field :id [:in id-chunk] {:fk_target_field_id fk-tid}))))--(defn- import-metadata!*- "Core logic for POST /metadata. See the endpoint doc for behavior.-- Runs a separate `t2/with-transaction` per matched target DB: each DB's tables,- fields, and same-DB FK references are imported in isolation so one DB's failure- does not roll back the rest, and memory usage is bounded by the largest single- DB's target-field-pathmap rather than the whole payload. After every per-DB- transaction has committed, a final transaction resolves `fk_target_field_id`- references that cross DB boundaries."- [{:keys [databases tables fields]}]- (let [db-by-key (build-db-lookup)- in-db->target (into {}- (keep (fn [{:keys [id name engine]}]- (when-some [tid (db-by-key [name (engine-name engine)])]- [id tid])))- databases)- missing-dbs (mapv #(select-keys % [:name :engine])- (remove #(in-db->target (:id %)) databases))- incoming-by-id (m/index-by :id fields)- table->db (into {} (map (juxt :id :db_id)) tables)- tables-by-db (group-by :db_id tables)- fields-by-db (group-by #(table->db (:table_id %)) fields)- ;; Tables/fields whose DB isn't matched, or fields whose table_id isn't in- ;; tables[], can't be placed — record as missing up front so per-DB loops- ;; only handle processable rows.- missing-tables (into []- (keep (fn [{:keys [db_id schema name]}]- (when-not (in-db->target db_id)- (cond-> {:db_id db_id :name name}- (some? schema) (assoc :schema schema)))))- tables)- missing-fields (into []- (keep (fn [{:keys [id table_id]}]- (let [incoming-db (table->db table_id)]- (when (or (nil? incoming-db)- (not (in-db->target incoming-db)))- {:table_id table_id- :path (incoming-field-path incoming-by-id id)}))))- fields)- global-state (atom {:tables {:matched 0 :created 0 :missing missing-tables}- :fields {:matched 0 :created 0 :missing missing-fields}- :in-fld->target {}- :failed-dbs []})]- (doseq [[incoming-db-id target-db-id] in-db->target]- (let [db-tables (get tables-by-db incoming-db-id [])- db-fields (get fields-by-db incoming-db-id [])- db-state (atom {:tables (:tables @global-state)- :fields (:fields @global-state)- :in-tbl->target {}- :in-fld->target (:in-fld->target @global-state)})]- (try- (t2/with-transaction [_conn]- (let [target-tbl-lookup (build-target-table-lookup [target-db-id])]- (import-tables! db-state db-tables target-db-id target-tbl-lookup))- (let [in-tbl->target (:in-tbl->target @db-state)- path-lookup (build-target-field-pathmap (vals in-tbl->target))]- (import-fields! db-state db-fields incoming-by-id in-tbl->target path-lookup))- (resolve-fk-references!- (fk-pairs-of-kind db-fields incoming-by-id table->db- (:in-fld->target @db-state) :same-db)))- (swap! global-state- (fn [g]- (-> g- (assoc :tables (:tables @db-state))- (assoc :fields (:fields @db-state))- (update :in-fld->target merge (:in-fld->target @db-state)))))- (catch Throwable t- (log/errorf t "POST /api/database/metadata: import failed for db %s (target %s)"- incoming-db-id target-db-id)- (swap! global-state update :failed-dbs conj- {:id incoming-db-id :target target-db-id :error (.getMessage t)})))))- (let [cross-db-pairs (fk-pairs-of-kind fields incoming-by-id table->db- (:in-fld->target @global-state) :cross-db)]- (when (seq cross-db-pairs)- (t2/with-transaction [_conn]- (resolve-fk-references! cross-db-pairs))))- (let [{tbl-report :tables fld-report :fields failed-dbs :failed-dbs} @global-state]- {:databases (cond-> {:matched (count in-db->target)- :missing missing-dbs}- (seq failed-dbs) (assoc :failed failed-dbs))- :tables tbl-report- :fields fld-report})))--(mr/def ::metadata-import-report- [:map- [:databases [:map- [:matched :int]- [:missing [:sequential :map]]- [:failed {:optional true} [:sequential :map]]]]- [:tables [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]- [:fields [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]])--(api.macros/defendpoint :post "/metadata"- :- ::metadata-import-report- "Import database/table/field metadata previously exported from `GET /api/database/metadata`.-- Entities are matched by natural key — databases by `(name, engine)`, tables by- `(database, schema, name)`, fields by `(table, parent-path, name)` — so the numeric ids- in the payload are only used to link fields to their tables within the request.-- Tables and fields that don't exist on the target are created when their parent- (database for tables; table for fields) is present on the target. Databases are- not auto-created — missing databases are reported instead. Field `fk_target_field_id`- is re-resolved through the natural-key lookup after all fields exist.-- For matched entities, only user-editable metadata is written: table `description`;- field `description`, `semantic_type`, `coercion_strategy`, `effective_type`, and- `fk_target_field_id`. For newly-created fields, `base_type` and `database_type` are- also populated from the payload. Keys absent from the payload (including null values)- are left untouched on matched entities.-- Processing is isolated per target database: each matched DB imports in its own- transaction so a failure on one DB does not roll back the others. DBs whose- transaction failed appear under `databases.failed` in the response along with- the error message; every other DB's tables, fields, and same-DB `fk_target_field_id`- references still commit. Cross-database `fk_target_field_id` references are- resolved in a final pass after all per-DB transactions have committed.-- Returns counts of matched + created entities per type, a list of entities in the- payload that could not be placed (their parent was missing on the target), and,- when any DB failed, a `databases.failed` list naming each failed DB."- [_route-params- _query-params- body :- ::databases-metadata-response]- (api/check-superuser)- (import-metadata!* body))-;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbersdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex a8af32d5b74..c80ac3e959b 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -874,360 +874,6 @@(is (nil? (m/find-first (comp #{t-id} :id) tables)))(is (nil? (m/find-first (comp #{f-id} :id) fields)))))))-(deftest databases-metadata-import-test- (testing "POST /api/database/metadata"- (mt/with-temp [:model/Database {db-id :id} {:name "import-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "orders" :schema "PUBLIC"- :description "original"}- :model/Field {pk-id :id} {:table_id t-id :name "id" :base_type :type/Integer- :database_type "BIGINT"}- :model/Field {fk-id :id} {:table_id t-id :name "order_id" :base_type :type/Integer- :database_type "BIGINT"}]- (testing "matched entities are updated; missing tables/fields are created when parent exists"- ;; Payload carries ids from "another" instance — here we reuse our own ids, but the- ;; endpoint matches by natural key regardless of what the numeric ids are.- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}- {:id 9999 :name "does-not-exist" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"- :description "updated via import"}- {:id 9998 :db_id db-id :name "new_table" :schema "PUBLIC"- :description "created via import"}]- :fields [{:id pk-id :table_id t-id :name "id"- :base_type "type/Integer" :database_type "BIGINT"- :semantic_type "type/PK"- :description "primary key"}- {:id fk-id :table_id t-id :name "order_id"- :base_type "type/Integer" :database_type "BIGINT"- :semantic_type "type/FK"- :fk_target_field_id pk-id}- {:id 9997 :table_id t-id :name "new_field"- :base_type "type/Integer" :database_type "INT"- :description "created via import"- :semantic_type "type/Quantity"}- {:id 9996 :table_id 9998 :name "new_table_field"- :base_type "type/Text" :database_type "VARCHAR"}]}- report (mt/user-http-request :crowberto :post 200- "database/metadata" payload)]- (is (=? {:databases {:matched 1 :missing [{:name "does-not-exist"}]}- :tables {:matched 1 :created 1 :missing []}- :fields {:matched 2 :created 2 :missing []}}- report))- (is (= "updated via import" (t2/select-one-fn :description :model/Table :id t-id)))- (is (= :type/PK (t2/select-one-fn :semantic_type :model/Field :id pk-id)))- (is (= "primary key" (t2/select-one-fn :description :model/Field :id pk-id)))- (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))- (testing "new table was created under the matched database"- (let [new-tbl (t2/select-one :model/Table :db_id db-id :name "new_table")]- (is (some? new-tbl))- (is (= "created via import" (:description new-tbl)))- (is (true? (:active new-tbl)))))- (testing "new field was created under the existing table"- (let [new-fld (t2/select-one :model/Field :table_id t-id :name "new_field")]- (is (some? new-fld))- (is (= :type/Integer (:base_type new-fld)))- (is (= "INT" (:database_type new-fld)))- (is (= "created via import" (:description new-fld)))- (is (= :type/Quantity (:semantic_type new-fld)))))- (testing "new field was created under a newly-created table"- (let [new-tbl-id (t2/select-one-pk :model/Table :db_id db-id :name "new_table")]- (is (some? (t2/select-one :model/Field :table_id new-tbl-id :name "new_table_field")))))))-- (testing "fields whose database is missing on the target are reported as missing"- (let [payload {:databases [{:id 9999 :name "does-not-exist" :engine "h2"}]- :tables [{:id 9998 :db_id 9999 :name "x" :schema "PUBLIC"}]- :fields [{:id 9997 :table_id 9998 :name "y"- :base_type "type/Integer" :database_type "INTEGER"}]}- report (mt/user-http-request :crowberto :post 200- "database/metadata" payload)]- (is (=? {:tables {:matched 0 :created 0 :missing [{:name "x"}]}- :fields {:matched 0 :created 0 :missing [{:path ["y"]}]}}- report))))-- (testing "base_type and database_type are never overwritten"- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]- :fields [{:id pk-id :table_id t-id :name "id"- :base_type "type/Text" :database_type "TEXT"- :description "still a pk"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (is (= :type/Integer (t2/select-one-fn :base_type :model/Field :id pk-id)))- (is (= "BIGINT" (t2/select-one-fn :database_type :model/Field :id pk-id)))))-- (testing "nested fields are matched by parent path"- (mt/with-temp [:model/Field {parent-id :id} {:table_id t-id :name "payload"- :base_type :type/JSON :database_type "JSON"}- :model/Field {child-id :id} {:table_id t-id :parent_id parent-id :name "amount"- :base_type :type/Integer :database_type "BIGINT"}]- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-id :name "payload"- :base_type "type/JSON" :database_type "JSON"}- {:id 2 :table_id t-id :parent_id 1 :name "amount"- :base_type "type/Integer" :database_type "BIGINT"- :description "nested description"- :semantic_type "type/Quantity"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (is (= "nested description" (t2/select-one-fn :description :model/Field :id child-id)))- (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id child-id))))))-- (testing "fields with the same leaf name at different parent paths are matched independently"- ;; A root-level `amount` and a nested `payload.amount` coexist on the same table.- ;; Matching by full parent path must update each without clobbering the other.- (mt/with-temp [:model/Field {root-amount-id :id} {:table_id t-id :name "amount"- :base_type :type/Integer :database_type "BIGINT"}- :model/Field {parent-id :id} {:table_id t-id :name "payload"- :base_type :type/JSON :database_type "JSON"}- :model/Field {nested-amount-id :id} {:table_id t-id :parent_id parent-id :name "amount"- :base_type :type/Integer :database_type "BIGINT"}]- (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]- :fields [{:id 10 :table_id t-id :name "amount"- :base_type "type/Integer" :database_type "BIGINT"- :description "root amount"- :semantic_type "type/Quantity"}- {:id 11 :table_id t-id :name "payload"- :base_type "type/JSON" :database_type "JSON"}- {:id 12 :table_id t-id :parent_id 11 :name "amount"- :base_type "type/Integer" :database_type "BIGINT"- :description "nested amount"- :semantic_type "type/Currency"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (is (= "root amount" (t2/select-one-fn :description :model/Field :id root-amount-id)))- (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id root-amount-id)))- (is (= "nested amount" (t2/select-one-fn :description :model/Field :id nested-amount-id)))- (is (= :type/Currency (t2/select-one-fn :semantic_type :model/Field :id nested-amount-id))))))-- (testing "non-superusers are rejected"- (mt/user-http-request :rasta :post 403 "database/metadata"- {:databases [] :tables [] :fields []})))))--(deftest databases-metadata-import-nested-fields-test- (testing "POST /api/database/metadata — nested fields with shared leaf names under siblings"- (mt/with-temp [:model/Database {db-id :id} {:name "nested-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "nested-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]- :fields [{:id 100 :table_id t-id :name "wind"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 101 :table_id t-id :parent_id 100 :name "value"- :base_type "type/Float" :database_type "NULL"}- {:id 102 :table_id t-id :name "temp"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 103 :table_id t-id :parent_id 102 :name "value"- :base_type "type/Float" :database_type "NULL"}]}- report (mt/user-http-request :crowberto :post 200- "database/metadata" payload)- by-name (->> (t2/select [:model/Field :id :name :parent_id] :table_id t-id)- (group-by :name))- wind-id (:id (first (get by-name "wind")))- temp-id (:id (first (get by-name "temp")))- values (sort-by :parent_id (get by-name "value"))]- (is (=? {:databases {:matched 1 :missing []}- :tables {:matched 1 :created 0 :missing []}- :fields {:matched 0 :created 4 :missing []}}- report))- (is (= 4 (t2/count :model/Field :table_id t-id)))- (is (= 2 (count values)))- (testing "each value row points at its own parent, not NULL"- (is (= #{wind-id temp-id}- (set (map :parent_id values))))))))-- (testing "POST /api/database/metadata — 3-level deep nesting with correct parent chain"- (mt/with-temp [:model/Database {db-id :id} {:name "nested-3-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "tree" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "nested-3-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "tree" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-id :name "a"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 2 :table_id t-id :parent_id 1 :name "b"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 3 :table_id t-id :parent_id 2 :name "c"- :base_type "type/Integer" :database_type "NULL"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (let [a (t2/select-one :model/Field :table_id t-id :name "a")- b (t2/select-one :model/Field :table_id t-id :name "b")- c (t2/select-one :model/Field :table_id t-id :name "c")]- (is (nil? (:parent_id a)))- (is (= (:id a) (:parent_id b)))- (is (= (:id b) (:parent_id c)))))))-- (testing "POST /api/database/metadata — new child attaches to a pre-existing matched parent"- (mt/with-temp [:model/Database {db-id :id} {:name "matched-parent-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}- :model/Field {p-id :id} {:table_id t-id :name "payload"- :base_type :type/Dictionary}]- (let [payload {:databases [{:id db-id :name "matched-parent-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]- :fields [{:id 10 :table_id t-id :name "payload"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 11 :table_id t-id :parent_id 10 :name "new_leaf"- :base_type "type/Integer" :database_type "NULL"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (let [child (t2/select-one :model/Field :table_id t-id :name "new_leaf")]- (is (some? child))- (is (= p-id (:parent_id child)))))))-- (testing "POST /api/database/metadata — nested-field import is idempotent on repeat"- (mt/with-temp [:model/Database {db-id :id} {:name "idempotency-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "idempotency-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]- :fields [{:id 100 :table_id t-id :name "wind"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 101 :table_id t-id :parent_id 100 :name "value"- :base_type "type/Float" :database_type "NULL"}- {:id 102 :table_id t-id :name "temp"- :base_type "type/Dictionary" :database_type "NULL"}- {:id 103 :table_id t-id :parent_id 102 :name "value"- :base_type "type/Float" :database_type "NULL"}]}- first-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- second-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]- (is (=? {:fields {:matched 0 :created 4}} first-report))- (is (=? {:fields {:matched 4 :created 0}} second-report))- (is (= 4 (t2/count :model/Field :table_id t-id)))))))--(deftest databases-metadata-import-edge-cases-test- (testing "POST /api/database/metadata — orphan parent_id (references a field not in payload) lands as root"- (mt/with-temp [:model/Database {db-id :id} {:name "orphan-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "orphan-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]- :fields [{:id 99 :table_id t-id :parent_id 9999 :name "orphan"- :base_type "type/Text" :database_type "TEXT"}]}- report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]- (is (=? {:fields {:created 1}} report))- (let [orphan (t2/select-one :model/Field :table_id t-id :name "orphan")]- (is (some? orphan))- (is (nil? (:parent_id orphan)))))))-- (testing "POST /api/database/metadata — omitted database_type falls back to the \"NULL\" sentinel"- ;; GET's `format-field-metadata` drops nil `database_type` via `m/assoc-some`,- ;; so a round-tripped payload from a MySQL app DB (nullable column) can omit- ;; the key. The sentinel keeps the INSERT valid under Postgres' NOT NULL.- (mt/with-temp [:model/Database {db-id :id} {:name "no-db-type-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "no-db-type-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-id :name "mongo_field"- :base_type "type/Text"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (is (= "NULL"- (t2/select-one-fn :database_type :model/Field :table_id t-id :name "mongo_field"))))))-- (testing "POST /api/database/metadata — cycle in incoming parent_id does not stack-overflow"- (mt/with-temp [:model/Database {db-id :id} {:name "cycle-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]- (let [payload {:databases [{:id db-id :name "cycle-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-id :parent_id 2 :name "a"- :base_type "type/Text" :database_type "TEXT"}- {:id 2 :table_id t-id :parent_id 1 :name "b"- :base_type "type/Text" :database_type "TEXT"}]}]- ;; Degenerate payload — only the status code is asserted; the rows' shape- ;; after a cycle-broken insert is intentionally unspecified.- (is (=? {:databases {:matched 1}}- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)))))))--(deftest databases-metadata-import-batching-test- (testing "POST /api/database/metadata — new-field INSERTs are chunked by import-batch-size"- (mt/with-temp [:model/Database {db-id :id} {:name "batch-db" :engine :h2}- :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]- (let [insert-calls (atom [])- orig-insert t2/insert-returning-pks!- payload {:databases [{:id db-id :name "batch-db" :engine "h2"}]- :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]- :fields (mapv (fn [idx]- {:id (+ 1000 idx)- :table_id t-id- :name (str "col_" idx)- :base_type "type/Integer"- :database_type "INTEGER"})- (range 7))}]- (with-redefs [api.database/import-batch-size 3- t2/insert-returning-pks! (fn [model rows]- (when (= model :model/Field)- (swap! insert-calls conj (count rows)))- (orig-insert model rows))]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload))- (is (= [3 3 1] @insert-calls))))))--(deftest databases-metadata-import-partial-failure-test- (testing "POST /api/database/metadata — one DB's failure does not roll back others"- (mt/with-temp [:model/Database {db-a-id :id} {:name "pf-db-a" :engine :h2}- :model/Database {db-b-id :id} {:name "pf-db-b" :engine :h2}- :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}- :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}]- (let [orig-import-fields (deref #'api.database/import-fields!)]- (with-redefs [api.database/import-fields! (fn [state fields incoming-by-id in-tbl->target path-lookup]- (if (some #(= t-b-id (:table_id %)) fields)- (throw (ex-info "injected failure for DB-B" {}))- (orig-import-fields state fields incoming-by-id- in-tbl->target path-lookup)))]- (let [payload {:databases [{:id db-a-id :name "pf-db-a" :engine "h2"}- {:id db-b-id :name "pf-db-b" :engine "h2"}]- :tables [{:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}- {:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-a-id :name "a_col"- :base_type "type/Integer" :database_type "INTEGER"}- {:id 2 :table_id t-b-id :name "b_col"- :base_type "type/Integer" :database_type "INTEGER"}]}- report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]- (is (=? {:databases {:matched 2- :missing []- :failed [{:id db-b-id :target db-b-id}]}}- report))- (testing "DB-A's field committed despite DB-B's failure"- (is (some? (t2/select-one :model/Field :table_id t-a-id :name "a_col"))))- (testing "DB-B's field did not commit (transaction rolled back)"- (is (nil? (t2/select-one :model/Field :table_id t-b-id :name "b_col")))))))))-- (testing "POST /api/database/metadata — pathmap is scoped to the current DB only"- (mt/with-temp [:model/Database {db-a-id :id} {:name "scope-db-a" :engine :h2}- :model/Database {db-b-id :id} {:name "scope-db-b" :engine :h2}- :model/Table {t-a-id :id} {:db_id db-a-id :name "t" :schema "PUBLIC"}- :model/Table {t-b-id :id} {:db_id db-b-id :name "t" :schema "PUBLIC"}]- (let [calls (atom [])- orig-build-pm (deref #'api.database/build-target-field-pathmap)]- (with-redefs [api.database/build-target-field-pathmap (fn [ids]- (swap! calls conj (set ids))- (orig-build-pm ids))]- (let [payload {:databases [{:id db-a-id :name "scope-db-a" :engine "h2"}]- :tables [{:id t-a-id :db_id db-a-id :name "t" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-a-id :name "c"- :base_type "type/Integer" :database_type "INTEGER"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload))- (testing "build-target-field-pathmap never saw db-b's table"- (is (every? #(not (contains? % t-b-id)) @calls)))- (testing "and was called with db-a's table"- (is (some #(contains? % t-a-id) @calls))))))))--(deftest databases-metadata-import-cross-db-fk-test- (testing "POST /api/database/metadata — cross-DB fk_target_field_id resolves after all DBs commit"- (mt/with-temp [:model/Database {db-a-id :id} {:name "xfk-db-a" :engine :h2}- :model/Database {db-b-id :id} {:name "xfk-db-b" :engine :h2}- :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}- :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}- :model/Field {pk-id :id} {:table_id t-a-id :name "id"- :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/PK}- :model/Field {fk-id :id} {:table_id t-b-id :name "a_id"- :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/FK}]- (let [payload {:databases [{:id db-b-id :name "xfk-db-b" :engine "h2"}- {:id db-a-id :name "xfk-db-a" :engine "h2"}]- :tables [{:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}- {:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}]- :fields [{:id fk-id :table_id t-b-id :name "a_id"- :base_type "type/Integer" :database_type "BIGINT"- :fk_target_field_id pk-id}- {:id pk-id :table_id t-a-id :name "id"- :base_type "type/Integer" :database_type "BIGINT"- :semantic_type "type/PK"}]}]- (mt/user-http-request :crowberto :post 200 "database/metadata" payload)- (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))))))-(deftest ^:parallel fetch-database-metadata-test(testing "GET /api/database/:id/metadata"(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)