The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix metadata import oom (#72930)
Aleksandr Lesnenkodcbe8d30e086
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 391ad2197e7..8567b2df84a 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -698,6 +698,12 @@;;; ----------------------------------------- POST /api/database/metadata -----------------------------------------+(def ^:private import-batch-size+ "Row batch size for bulk INSERTs and IN-list UPDATEs in POST /api/database/metadata.+ For an 8-column field row, 2000 × 8 = 16k prepared-statement parameters per+ statement — well under Postgres' 65535 cap and MySQL's default max_allowed_packet."+ 2000)+(defn- engine-name"Normalize engine (stored as string or keyword) to a string for lookup."[engine]@@ -738,13 +744,44 @@[[table_id (path field)] id]))rows))))+(def ^:private max-resolution-depth+ "Safety cap on `parent_id` chain walks (`incoming-field-path`, `resolution-depth`)+ so a malformed payload with cycles or deep chains can't stack-overflow or stall+ the import."+ 100)+(defn- incoming-field-path- "[name1 ... leaf] path of an incoming field, walking parent_id in the payload."+ "[name1 ... leaf] path of an incoming field, walking parent_id in the payload.+ Cycle-safe: if the parent chain loops or exceeds `max-resolution-depth` the walk+ terminates at that point, treating the field as effectively rooted there."[incoming-by-id field-id]- (when-some [{:keys [name parent_id]} (get incoming-by-id field-id)]- (if parent_id- (conj (incoming-field-path incoming-by-id parent_id) name)- [name])))+ (loop [id field-id, path (), seen #{}, depth 0]+ (if (or (contains? seen id) (>= depth max-resolution-depth))+ (vec path)+ (if-some [{:keys [name parent_id]} (get incoming-by-id id)]+ (let [path' (conj path name)]+ (if parent_id+ (recur parent_id path' (conj seen id) (inc depth))+ (vec path')))+ (vec path)))))++(defn- resolution-depth+ "Minimum insert-wave at which a new field can be written. Depth 0 means the+ field's parent is already resolvable on the target — nil parent, matched parent+ (already in `in-fld->target`), or orphan parent (payload references a parent that+ isn't in the `fields[]` array). Depth N means the parent is itself a new field+ that must be inserted at wave N-1 first. Cycles and chains deeper than+ `max-resolution-depth` fall back to 0 (root-level insert)."+ [incoming-by-id in-fld->target field-id]+ (loop [id field-id, depth 0, seen #{}]+ (if (or (>= depth max-resolution-depth) (contains? seen id))+ 0+ (let [{:keys [parent_id]} (get incoming-by-id id)]+ (cond+ (nil? parent_id) depth+ (contains? in-fld->target parent_id) depth+ (nil? (get incoming-by-id parent_id)) depth+ :else (recur parent_id (inc depth) (conj seen id)))))))(defn- new-table-row[target-db-id {:keys [schema name description]}]@@ -756,16 +793,17 @@(some? description) (assoc :description description)))(defn- new-field-row- "Row for inserting a new field. parent_id and fk_target_field_id are intentionally- omitted — they're set in the references pass after every field exists."+ "Row for inserting a new field. `parent_id` is resolved by `import-fields!` at+ INSERT time (see that function's docstring for why); `fk_target_field_id` is+ resolved in a later pass after every field exists."[target-tbl-id{:keys [name base_type description database_type effective_type semantic_type coercion_strategy]}]- (cond-> {:table_id target-tbl-id- :name name- :base_type (or base_type :type/*)- :active true}+ (cond-> {:table_id target-tbl-id+ :name name+ :base_type base_type+ :database_type (or database_type "NULL")+ :active true}(some? description) (assoc :description description)- (some? database_type) (assoc :database_type database_type)(some? effective_type) (assoc :effective_type effective_type)(some? semantic_type) (assoc :semantic_type semantic_type)(some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)))@@ -780,33 +818,33 @@(some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)(some? effective_type) (assoc :effective_type effective_type)))-(defn- import-tables!- "Phase 1. Match by (target-db, schema, name) against a full lookup of the target's- active tables; UPDATE matched rows' description, bulk INSERT the rest."- [state tables in-db->target]- (let [target-lookup (build-target-table-lookup (vals in-db->target))- to-insert (volatile! [])]- (doseq [{:keys [id db_id schema name description] :as tbl} tables]- (let [target-db-id (in-db->target db_id)- existing-id (when target-db-id (target-lookup [target-db-id schema name]))]- (cond- (nil? target-db-id)- (swap! state update-in [:tables :missing] conj- (cond-> {:db_id db_id :name name}- (some? schema) (assoc :schema schema)))+(defn- batched-insert-returning-pks!+ "INSERT `rows` in chunks of `import-batch-size`, returning a vector of pks in+ input order. `into []` drives the transducer to completion, so every chunk's+ `t2/insert-returning-pks!` — and its side effects — runs before this returns."+ [model rows]+ (into []+ (mapcat (fn [chunk] (t2/insert-returning-pks! model chunk)))+ (partition-all import-batch-size rows)))- existing-id- (do- (when (some? description)- (t2/update! :model/Table existing-id {:description description}))- (swap! state #(-> %- (update-in [:tables :matched] inc)- (assoc-in [:in-tbl->target id] existing-id))))-- :else- (vswap! to-insert conj [id (new-table-row target-db-id tbl)]))))+(defn- import-tables!+ "Phase 1 (per-DB). Match tables by (schema, name) against `target-lookup` (a+ `{[target-db-id schema name] -> target-table-id}` map already scoped to this DB);+ UPDATE matched rows' description per-row, chunked-INSERT the rest via+ `batched-insert-returning-pks!`."+ [state tables target-db-id target-lookup]+ (let [to-insert (volatile! [])]+ (doseq [{:keys [id schema name description] :as tbl} tables]+ (if-some [existing-id (target-lookup [target-db-id schema name])]+ (do+ (when (some? description)+ (t2/update! :model/Table existing-id {:description description}))+ (swap! state #(-> %+ (update-in [:tables :matched] inc)+ (assoc-in [:in-tbl->target id] existing-id))))+ (vswap! to-insert conj [id (new-table-row target-db-id tbl)])))(when-some [rows (seq @to-insert)]- (let [new-ids (t2/insert-returning-pks! :model/Table (mapv second rows))]+ (let [new-ids (batched-insert-returning-pks! :model/Table (mapv second rows))](swap! state (fn [s](reduce (fn [s [[incoming-id _] new-id]](-> s@@ -816,12 +854,20 @@(map vector rows new-ids))))))))(defn- import-fields!- "Phase 2. Match incoming fields against the target by full (table, parent-path, name)- using `build-target-field-pathmap`; UPDATE matched rows with editable metadata (minus- parent_id / fk_target_field_id), bulk INSERT the rest with parent_id left NULL."- [state fields incoming-by-id in-tbl->target]- (let [path-lookup (build-target-field-pathmap (vals in-tbl->target))- to-insert (volatile! [])]+ "Phase 2 (per-DB). Classify each incoming field as matched / missing / new, then+ insert new fields in depth-ordered waves so every child's parent_id is resolved+ at INSERT time. Matched rows get per-row UPDATE of editable metadata; parent_id+ on matched rows is never changed and fk_target_field_id is handled by+ `resolve-fk-references!`.++ Waves are required — not just convenient — because the `idx_unique_field` DB+ constraint is on `(name, table_id, COALESCE(parent_id, 0))`. Inserting children+ with `parent_id = NULL` would make every root-level row (and every child of a+ not-yet-inserted parent) share `unique_field_helper = 0`, so sibling nested+ leaves that share a leaf name collide. Writing `parent_id` at INSERT time keeps+ each such row under a distinct helper key."+ [state fields incoming-by-id in-tbl->target path-lookup]+ (let [to-insert (volatile! [])](doseq [{:keys [id table_id] :as fld} fields](let [target-tbl (in-tbl->target table_id)path (when target-tbl (incoming-field-path incoming-by-id id))@@ -841,37 +887,80 @@(assoc-in [:in-fld->target id] existing-id)))):else- (vswap! to-insert conj [id (new-field-row target-tbl fld)]))))- (when-some [rows (seq @to-insert)]- (let [new-ids (t2/insert-returning-pks! :model/Field (mapv second rows))]- (swap! state (fn [s]- (reduce (fn [s [[incoming-id _] new-id]]- (-> s- (update-in [:fields :created] inc)- (assoc-in [:in-fld->target incoming-id] new-id)- (update :created-fld-ids conj incoming-id)))- s- (map vector rows new-ids))))))))--(defn- resolve-field-references!- "Phase 3. Link parent_id and fk_target_field_id now that every resolvable field has a- target id. parent_id is only set on rows we just created — matched rows keep their- existing parent so we never restructure a nested field in place. fk_target_field_id is- user-editable and is updated for both matched and created rows."- [fields in-fld->target created-fld-ids]- (doseq [{:keys [id parent_id fk_target_field_id]} fields]- (let [tid (in-fld->target id)- patch (cond-> {}- (and (contains? created-fld-ids id) (some? parent_id))- (assoc :parent_id (in-fld->target parent_id))-- (some? fk_target_field_id)- (assoc :fk_target_field_id (in-fld->target fk_target_field_id)))]- (when (and tid (seq patch))- (t2/update! :model/Field tid patch)))))+ (vswap! to-insert conj [id fld target-tbl]))))+ (let [matched-map (:in-fld->target @state)+ by-depth (reduce (fn [acc [id :as triple]]+ (let [depth (resolution-depth incoming-by-id matched-map id)]+ (update acc depth (fnil conj []) triple)))+ (sorted-map)+ @to-insert)]+ (vreset! to-insert nil)+ (doseq [[_depth entries] by-depth]+ (let [in-fld->target (:in-fld->target @state)+ rows (mapv (fn [[_id fld target-tbl]]+ (let [base (new-field-row target-tbl fld)+ pid (get in-fld->target (:parent_id fld))]+ (cond-> base+ pid (assoc :parent_id pid))))+ entries)+ new-ids (batched-insert-returning-pks! :model/Field rows)]+ (swap! state (fn [s]+ (reduce (fn [s [[incoming-id _ _] new-id]]+ (-> s+ (update-in [:fields :created] inc)+ (assoc-in [:in-fld->target incoming-id] new-id)))+ s+ (map vector entries new-ids)))))))))++(defn- classify-fk+ "Classify a field's `fk_target_field_id` against the incoming payload:+ `:same-db` when the target resolves to the same incoming DB; `:cross-db` when+ it resolves to a different DB; `nil` when the field has no FK, or the target+ isn't in the payload, or its DB can't be determined."+ [incoming-by-id table->db {:keys [fk_target_field_id table_id]}]+ (when-some [target (get incoming-by-id fk_target_field_id)]+ (let [src (table->db table_id)+ dst (table->db (:table_id target))]+ (cond+ (nil? dst) nil+ (= src dst) :same-db+ :else :cross-db))))++(defn- fk-pairs-of-kind+ "Return `[[self-tid fk-tid] ...]` for every field in `fields` whose fk classifies+ as `kind` and whose self-id AND fk target both resolved in `in-fld->target`."+ [fields incoming-by-id table->db in-fld->target kind]+ (into []+ (keep (fn [{:keys [id fk_target_field_id] :as fld}]+ (when (= kind (classify-fk incoming-by-id table->db fld))+ (let [self-tid (in-fld->target id)+ fk-tid (in-fld->target fk_target_field_id)]+ (when (and self-tid fk-tid) [self-tid fk-tid])))))+ fields))++(defn- resolve-fk-references!+ "Phase 3. Write `fk_target_field_id` for each `[self-tid fk-tid]` pair, grouping+ by `fk-tid` so many sources pointing at the same target collapse into a single+ `UPDATE ... WHERE id IN (...)`, chunked to `import-batch-size` ids per statement.+ `parent_id` is NOT handled here — it's written at INSERT time by `import-fields!`."+ [pairs]+ (let [buckets (reduce (fn [acc [self-tid fk-tid]]+ (update acc fk-tid (fnil conj []) self-tid))+ {}+ pairs)]+ (doseq [[fk-tid ids] buckets+ id-chunk (partition-all import-batch-size ids)]+ (t2/update! :model/Field :id [:in id-chunk] {:fk_target_field_id fk-tid}))))(defn- import-metadata!*- "Core logic for POST /metadata. See the endpoint doc for behavior."+ "Core logic for POST /metadata. See the endpoint doc for behavior.++ Runs a separate `t2/with-transaction` per matched target DB: each DB's tables,+ fields, and same-DB FK references are imported in isolation so one DB's failure+ does not roll back the rest, and memory usage is bounded by the largest single+ DB's target-field-pathmap rather than the whole payload. After every per-DB+ transaction has committed, a final transaction resolves `fk_target_field_id`+ references that cross DB boundaries."[{:keys [databases tables fields]}](let [db-by-key (build-db-lookup)in-db->target (into {}@@ -882,24 +971,76 @@missing-dbs (mapv #(select-keys % [:name :engine])(remove #(in-db->target (:id %)) databases))incoming-by-id (m/index-by :id fields)- state (atom {:tables {:matched 0 :created 0 :missing []}- :fields {:matched 0 :created 0 :missing []}- :in-tbl->target {}- :in-fld->target {}- :created-fld-ids #{}})]- (t2/with-transaction [_conn]- (import-tables! state tables in-db->target)- (import-fields! state fields incoming-by-id (:in-tbl->target @state))- (let [{:keys [in-fld->target created-fld-ids]} @state]- (resolve-field-references! fields in-fld->target created-fld-ids)))- (let [{:keys [tables fields]} @state]- {:databases {:matched (count in-db->target) :missing missing-dbs}- :tables tables- :fields fields})))+ table->db (into {} (map (juxt :id :db_id)) tables)+ tables-by-db (group-by :db_id tables)+ fields-by-db (group-by #(table->db (:table_id %)) fields)+ ;; Tables/fields whose DB isn't matched, or fields whose table_id isn't in+ ;; tables[], can't be placed — record as missing up front so per-DB loops+ ;; only handle processable rows.+ missing-tables (into []+ (keep (fn [{:keys [db_id schema name]}]+ (when-not (in-db->target db_id)+ (cond-> {:db_id db_id :name name}+ (some? schema) (assoc :schema schema)))))+ tables)+ missing-fields (into []+ (keep (fn [{:keys [id table_id]}]+ (let [incoming-db (table->db table_id)]+ (when (or (nil? incoming-db)+ (not (in-db->target incoming-db)))+ {:table_id table_id+ :path (incoming-field-path incoming-by-id id)}))))+ fields)+ global-state (atom {:tables {:matched 0 :created 0 :missing missing-tables}+ :fields {:matched 0 :created 0 :missing missing-fields}+ :in-fld->target {}+ :failed-dbs []})]+ (doseq [[incoming-db-id target-db-id] in-db->target]+ (let [db-tables (get tables-by-db incoming-db-id [])+ db-fields (get fields-by-db incoming-db-id [])+ db-state (atom {:tables (:tables @global-state)+ :fields (:fields @global-state)+ :in-tbl->target {}+ :in-fld->target (:in-fld->target @global-state)})]+ (try+ (t2/with-transaction [_conn]+ (let [target-tbl-lookup (build-target-table-lookup [target-db-id])]+ (import-tables! db-state db-tables target-db-id target-tbl-lookup))+ (let [in-tbl->target (:in-tbl->target @db-state)+ path-lookup (build-target-field-pathmap (vals in-tbl->target))]+ (import-fields! db-state db-fields incoming-by-id in-tbl->target path-lookup))+ (resolve-fk-references!+ (fk-pairs-of-kind db-fields incoming-by-id table->db+ (:in-fld->target @db-state) :same-db)))+ (swap! global-state+ (fn [g]+ (-> g+ (assoc :tables (:tables @db-state))+ (assoc :fields (:fields @db-state))+ (update :in-fld->target merge (:in-fld->target @db-state)))))+ (catch Throwable t+ (log/errorf t "POST /api/database/metadata: import failed for db %s (target %s)"+ incoming-db-id target-db-id)+ (swap! global-state update :failed-dbs conj+ {:id incoming-db-id :target target-db-id :error (.getMessage t)})))))+ (let [cross-db-pairs (fk-pairs-of-kind fields incoming-by-id table->db+ (:in-fld->target @global-state) :cross-db)]+ (when (seq cross-db-pairs)+ (t2/with-transaction [_conn]+ (resolve-fk-references! cross-db-pairs))))+ (let [{tbl-report :tables fld-report :fields failed-dbs :failed-dbs} @global-state]+ {:databases (cond-> {:matched (count in-db->target)+ :missing missing-dbs}+ (seq failed-dbs) (assoc :failed failed-dbs))+ :tables tbl-report+ :fields fld-report})))(mr/def ::metadata-import-report[:map- [:databases [:map [:matched :int] [:missing [:sequential :map]]]]+ [:databases [:map+ [:matched :int]+ [:missing [:sequential :map]]+ [:failed {:optional true} [:sequential :map]]]][:tables [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]][:fields [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]])@@ -922,8 +1063,16 @@also populated from the payload. Keys absent from the payload (including null values)are left untouched on matched entities.- Returns counts of matched + created entities per type plus a list of entities in the- payload that could not be placed (their parent was missing on the target)."+ Processing is isolated per target database: each matched DB imports in its own+ transaction so a failure on one DB does not roll back the others. DBs whose+ transaction failed appear under `databases.failed` in the response along with+ the error message; every other DB's tables, fields, and same-DB `fk_target_field_id`+ references still commit. Cross-database `fk_target_field_id` references are+ resolved in a final pass after all per-DB transactions have committed.++ Returns counts of matched + created entities per type, a list of entities in the+ payload that could not be placed (their parent was missing on the target), and,+ when any DB failed, a `databases.failed` list naming each failed DB."[_route-params_query-paramsbody :- ::databases-metadata-response]diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 378c8571420..8baf6f45bf9 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -875,7 +875,8 @@(testing "fields whose database is missing on the target are reported as missing"(let [payload {:databases [{:id 9999 :name "does-not-exist" :engine "h2"}]:tables [{:id 9998 :db_id 9999 :name "x" :schema "PUBLIC"}]- :fields [{:id 9997 :table_id 9998 :name "y" :base_type "type/Integer"}]}+ :fields [{:id 9997 :table_id 9998 :name "y"+ :base_type "type/Integer" :database_type "INTEGER"}]}report (mt/user-http-request :crowberto :post 200"database/metadata" payload)](is (=? {:tables {:matched 0 :created 0 :missing [{:name "x"}]}@@ -899,9 +900,10 @@:base_type :type/Integer :database_type "BIGINT"}](let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]:tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]- :fields [{:id 1 :table_id t-id :name "payload" :base_type "type/JSON"}+ :fields [{:id 1 :table_id t-id :name "payload"+ :base_type "type/JSON" :database_type "JSON"}{:id 2 :table_id t-id :parent_id 1 :name "amount"- :base_type "type/Integer"+ :base_type "type/Integer" :database_type "BIGINT":description "nested description":semantic_type "type/Quantity"}]}](mt/user-http-request :crowberto :post 200 "database/metadata" payload)@@ -920,12 +922,13 @@(let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]:tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]:fields [{:id 10 :table_id t-id :name "amount"- :base_type "type/Integer"+ :base_type "type/Integer" :database_type "BIGINT":description "root amount":semantic_type "type/Quantity"}- {:id 11 :table_id t-id :name "payload" :base_type "type/JSON"}+ {:id 11 :table_id t-id :name "payload"+ :base_type "type/JSON" :database_type "JSON"}{:id 12 :table_id t-id :parent_id 11 :name "amount"- :base_type "type/Integer"+ :base_type "type/Integer" :database_type "BIGINT":description "nested amount":semantic_type "type/Currency"}]}](mt/user-http-request :crowberto :post 200 "database/metadata" payload)@@ -938,6 +941,233 @@(mt/user-http-request :rasta :post 403 "database/metadata"{:databases [] :tables [] :fields []})))))+(deftest databases-metadata-import-nested-fields-test+ (testing "POST /api/database/metadata — nested fields with shared leaf names under siblings"+ (mt/with-temp [:model/Database {db-id :id} {:name "nested-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "nested-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]+ :fields [{:id 100 :table_id t-id :name "wind"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 101 :table_id t-id :parent_id 100 :name "value"+ :base_type "type/Float" :database_type "NULL"}+ {:id 102 :table_id t-id :name "temp"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 103 :table_id t-id :parent_id 102 :name "value"+ :base_type "type/Float" :database_type "NULL"}]}+ report (mt/user-http-request :crowberto :post 200+ "database/metadata" payload)+ by-name (->> (t2/select [:model/Field :id :name :parent_id] :table_id t-id)+ (group-by :name))+ wind-id (:id (first (get by-name "wind")))+ temp-id (:id (first (get by-name "temp")))+ values (sort-by :parent_id (get by-name "value"))]+ (is (=? {:databases {:matched 1 :missing []}+ :tables {:matched 1 :created 0 :missing []}+ :fields {:matched 0 :created 4 :missing []}}+ report))+ (is (= 4 (t2/count :model/Field :table_id t-id)))+ (is (= 2 (count values)))+ (testing "each value row points at its own parent, not NULL"+ (is (= #{wind-id temp-id}+ (set (map :parent_id values))))))))++ (testing "POST /api/database/metadata — 3-level deep nesting with correct parent chain"+ (mt/with-temp [:model/Database {db-id :id} {:name "nested-3-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "tree" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "nested-3-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "tree" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-id :name "a"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 2 :table_id t-id :parent_id 1 :name "b"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 3 :table_id t-id :parent_id 2 :name "c"+ :base_type "type/Integer" :database_type "NULL"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (let [a (t2/select-one :model/Field :table_id t-id :name "a")+ b (t2/select-one :model/Field :table_id t-id :name "b")+ c (t2/select-one :model/Field :table_id t-id :name "c")]+ (is (nil? (:parent_id a)))+ (is (= (:id a) (:parent_id b)))+ (is (= (:id b) (:parent_id c)))))))++ (testing "POST /api/database/metadata — new child attaches to a pre-existing matched parent"+ (mt/with-temp [:model/Database {db-id :id} {:name "matched-parent-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}+ :model/Field {p-id :id} {:table_id t-id :name "payload"+ :base_type :type/Dictionary}]+ (let [payload {:databases [{:id db-id :name "matched-parent-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]+ :fields [{:id 10 :table_id t-id :name "payload"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 11 :table_id t-id :parent_id 10 :name "new_leaf"+ :base_type "type/Integer" :database_type "NULL"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (let [child (t2/select-one :model/Field :table_id t-id :name "new_leaf")]+ (is (some? child))+ (is (= p-id (:parent_id child)))))))++ (testing "POST /api/database/metadata — nested-field import is idempotent on repeat"+ (mt/with-temp [:model/Database {db-id :id} {:name "idempotency-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "obs" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "idempotency-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "obs" :schema "PUBLIC"}]+ :fields [{:id 100 :table_id t-id :name "wind"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 101 :table_id t-id :parent_id 100 :name "value"+ :base_type "type/Float" :database_type "NULL"}+ {:id 102 :table_id t-id :name "temp"+ :base_type "type/Dictionary" :database_type "NULL"}+ {:id 103 :table_id t-id :parent_id 102 :name "value"+ :base_type "type/Float" :database_type "NULL"}]}+ first-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ second-report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]+ (is (=? {:fields {:matched 0 :created 4}} first-report))+ (is (=? {:fields {:matched 4 :created 0}} second-report))+ (is (= 4 (t2/count :model/Field :table_id t-id)))))))++(deftest databases-metadata-import-edge-cases-test+ (testing "POST /api/database/metadata — orphan parent_id (references a field not in payload) lands as root"+ (mt/with-temp [:model/Database {db-id :id} {:name "orphan-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "orphan-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]+ :fields [{:id 99 :table_id t-id :parent_id 9999 :name "orphan"+ :base_type "type/Text" :database_type "TEXT"}]}+ report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]+ (is (=? {:fields {:created 1}} report))+ (let [orphan (t2/select-one :model/Field :table_id t-id :name "orphan")]+ (is (some? orphan))+ (is (nil? (:parent_id orphan)))))))++ (testing "POST /api/database/metadata — omitted database_type falls back to the \"NULL\" sentinel"+ ;; GET's `format-field-metadata` drops nil `database_type` via `m/assoc-some`,+ ;; so a round-tripped payload from a MySQL app DB (nullable column) can omit+ ;; the key. The sentinel keeps the INSERT valid under Postgres' NOT NULL.+ (mt/with-temp [:model/Database {db-id :id} {:name "no-db-type-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "no-db-type-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-id :name "mongo_field"+ :base_type "type/Text"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (is (= "NULL"+ (t2/select-one-fn :database_type :model/Field :table_id t-id :name "mongo_field"))))))++ (testing "POST /api/database/metadata — cycle in incoming parent_id does not stack-overflow"+ (mt/with-temp [:model/Database {db-id :id} {:name "cycle-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]+ (let [payload {:databases [{:id db-id :name "cycle-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-id :parent_id 2 :name "a"+ :base_type "type/Text" :database_type "TEXT"}+ {:id 2 :table_id t-id :parent_id 1 :name "b"+ :base_type "type/Text" :database_type "TEXT"}]}]+ ;; Degenerate payload — only the status code is asserted; the rows' shape+ ;; after a cycle-broken insert is intentionally unspecified.+ (is (=? {:databases {:matched 1}}+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)))))))++(deftest databases-metadata-import-batching-test+ (testing "POST /api/database/metadata — new-field INSERTs are chunked by import-batch-size"+ (mt/with-temp [:model/Database {db-id :id} {:name "batch-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "t" :schema "PUBLIC"}]+ (let [insert-calls (atom [])+ orig-insert t2/insert-returning-pks!+ payload {:databases [{:id db-id :name "batch-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "t" :schema "PUBLIC"}]+ :fields (mapv (fn [idx]+ {:id (+ 1000 idx)+ :table_id t-id+ :name (str "col_" idx)+ :base_type "type/Integer"+ :database_type "INTEGER"})+ (range 7))}]+ (with-redefs [api.database/import-batch-size 3+ t2/insert-returning-pks! (fn [model rows]+ (when (= model :model/Field)+ (swap! insert-calls conj (count rows)))+ (orig-insert model rows))]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload))+ (is (= [3 3 1] @insert-calls))))))++(deftest databases-metadata-import-partial-failure-test+ (testing "POST /api/database/metadata — one DB's failure does not roll back others"+ (mt/with-temp [:model/Database {db-a-id :id} {:name "pf-db-a" :engine :h2}+ :model/Database {db-b-id :id} {:name "pf-db-b" :engine :h2}+ :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}+ :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}]+ (let [orig-import-fields (deref #'api.database/import-fields!)]+ (with-redefs [api.database/import-fields! (fn [state fields incoming-by-id in-tbl->target path-lookup]+ (if (some #(= t-b-id (:table_id %)) fields)+ (throw (ex-info "injected failure for DB-B" {}))+ (orig-import-fields state fields incoming-by-id+ in-tbl->target path-lookup)))]+ (let [payload {:databases [{:id db-a-id :name "pf-db-a" :engine "h2"}+ {:id db-b-id :name "pf-db-b" :engine "h2"}]+ :tables [{:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}+ {:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-a-id :name "a_col"+ :base_type "type/Integer" :database_type "INTEGER"}+ {:id 2 :table_id t-b-id :name "b_col"+ :base_type "type/Integer" :database_type "INTEGER"}]}+ report (mt/user-http-request :crowberto :post 200 "database/metadata" payload)]+ (is (=? {:databases {:matched 2+ :missing []+ :failed [{:id db-b-id :target db-b-id}]}}+ report))+ (testing "DB-A's field committed despite DB-B's failure"+ (is (some? (t2/select-one :model/Field :table_id t-a-id :name "a_col"))))+ (testing "DB-B's field did not commit (transaction rolled back)"+ (is (nil? (t2/select-one :model/Field :table_id t-b-id :name "b_col")))))))))++ (testing "POST /api/database/metadata — pathmap is scoped to the current DB only"+ (mt/with-temp [:model/Database {db-a-id :id} {:name "scope-db-a" :engine :h2}+ :model/Database {db-b-id :id} {:name "scope-db-b" :engine :h2}+ :model/Table {t-a-id :id} {:db_id db-a-id :name "t" :schema "PUBLIC"}+ :model/Table {t-b-id :id} {:db_id db-b-id :name "t" :schema "PUBLIC"}]+ (let [calls (atom [])+ orig-build-pm (deref #'api.database/build-target-field-pathmap)]+ (with-redefs [api.database/build-target-field-pathmap (fn [ids]+ (swap! calls conj (set ids))+ (orig-build-pm ids))]+ (let [payload {:databases [{:id db-a-id :name "scope-db-a" :engine "h2"}]+ :tables [{:id t-a-id :db_id db-a-id :name "t" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-a-id :name "c"+ :base_type "type/Integer" :database_type "INTEGER"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload))+ (testing "build-target-field-pathmap never saw db-b's table"+ (is (every? #(not (contains? % t-b-id)) @calls)))+ (testing "and was called with db-a's table"+ (is (some #(contains? % t-a-id) @calls))))))))++(deftest databases-metadata-import-cross-db-fk-test+ (testing "POST /api/database/metadata — cross-DB fk_target_field_id resolves after all DBs commit"+ (mt/with-temp [:model/Database {db-a-id :id} {:name "xfk-db-a" :engine :h2}+ :model/Database {db-b-id :id} {:name "xfk-db-b" :engine :h2}+ :model/Table {t-a-id :id} {:db_id db-a-id :name "ta" :schema "PUBLIC"}+ :model/Table {t-b-id :id} {:db_id db-b-id :name "tb" :schema "PUBLIC"}+ :model/Field {pk-id :id} {:table_id t-a-id :name "id"+ :base_type :type/Integer+ :database_type "BIGINT"+ :semantic_type :type/PK}+ :model/Field {fk-id :id} {:table_id t-b-id :name "a_id"+ :base_type :type/Integer+ :database_type "BIGINT"+ :semantic_type :type/FK}]+ (let [payload {:databases [{:id db-b-id :name "xfk-db-b" :engine "h2"}+ {:id db-a-id :name "xfk-db-a" :engine "h2"}]+ :tables [{:id t-b-id :db_id db-b-id :name "tb" :schema "PUBLIC"}+ {:id t-a-id :db_id db-a-id :name "ta" :schema "PUBLIC"}]+ :fields [{:id fk-id :table_id t-b-id :name "a_id"+ :base_type "type/Integer" :database_type "BIGINT"+ :fk_target_field_id pk-id}+ {:id pk-id :table_id t-a-id :name "id"+ :base_type "type/Integer" :database_type "BIGINT"+ :semantic_type "type/PK"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))))))+(deftest ^:parallel fetch-database-metadata-test(testing "GET /api/database/:id/metadata"(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)