The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
import json metadata (#72729)
Aleksandr Lesnenko3564f8662e16
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 88252024ddc..977354101ad 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -656,6 +656,240 @@(streaming-response {:content-type "application/json; charset=utf-8"} [os _](write-databases-metadata! os)))+;;; ----------------------------------------- POST /api/database/metadata -----------------------------------------++(defn- engine-name+ "Normalize engine (stored as string or keyword) to a string for lookup."+ [engine]+ (when engine (name engine)))++(defn- build-db-lookup+ "Map of [name engine-string] -> target database id."+ []+ (into {}+ (map (juxt (juxt :name (comp engine-name :engine)) :id))+ (t2/select [:model/Database :id :name :engine])))++(defn- build-target-table-lookup+ "{[target-db-id schema table-name] -> target-table-id} for every active table in the given dbs."+ [target-db-ids]+ (if (empty? target-db-ids)+ {}+ (into {}+ (map (juxt (juxt :db_id :schema :name) :id))+ (t2/select [:model/Table :id :db_id :schema :name]+ :db_id [:in target-db-ids] :active true))))++(defn- build-target-field-pathmap+ "{[target-table-id [name1 ... leaf]] -> target-field-id} for every active field under+ the given target tables. Paths are produced by walking `parent_id` on the target side."+ [target-table-ids]+ (if (empty? target-table-ids)+ {}+ (let [rows (t2/select [:model/Field :id :table_id :parent_id :name]+ :table_id [:in target-table-ids] :active true)+ by-id (into {} (map (juxt :id identity)) rows)+ path (fn path [field]+ (if-some [pid (:parent_id field)]+ (conj (path (get by-id pid)) (:name field))+ [(:name field)]))]+ (into {}+ (map (fn [{:keys [id table_id] :as field}]+ [[table_id (path field)] id]))+ rows))))++(defn- incoming-field-path+ "[name1 ... leaf] path of an incoming field, walking parent_id in the payload."+ [incoming-by-id field-id]+ (when-some [{:keys [name parent_id]} (get incoming-by-id field-id)]+ (if parent_id+ (conj (incoming-field-path incoming-by-id parent_id) name)+ [name])))++(defn- new-table-row+ [target-db-id {:keys [schema name description]}]+ (cond-> {:db_id target-db-id+ :name name+ :schema schema+ :active true+ :initial_sync_status "complete"}+ (some? description) (assoc :description description)))++(defn- new-field-row+ "Row for inserting a new field. parent_id and fk_target_field_id are intentionally+ omitted — they're set in the references pass after every field exists."+ [target-tbl-id+ {:keys [name base_type description database_type effective_type semantic_type coercion_strategy]}]+ (cond-> {:table_id target-tbl-id+ :name name+ :base_type (or base_type :type/*)+ :active true}+ (some? description) (assoc :description description)+ (some? database_type) (assoc :database_type database_type)+ (some? effective_type) (assoc :effective_type effective_type)+ (some? semantic_type) (assoc :semantic_type semantic_type)+ (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)))++(defn- matched-field-patch+ "Writable keys for a matched field — excludes parent_id (never re-parent matched rows)+ and fk_target_field_id (handled in the references pass)."+ [{:keys [description semantic_type coercion_strategy effective_type]}]+ (cond-> {}+ (some? description) (assoc :description description)+ (some? semantic_type) (assoc :semantic_type semantic_type)+ (some? coercion_strategy) (assoc :coercion_strategy coercion_strategy)+ (some? effective_type) (assoc :effective_type effective_type)))++(defn- import-tables!+ "Phase 1. Match by (target-db, schema, name) against a full lookup of the target's+ active tables; UPDATE matched rows' description, bulk INSERT the rest."+ [state tables in-db->target]+ (let [target-lookup (build-target-table-lookup (vals in-db->target))+ to-insert (volatile! [])]+ (doseq [{:keys [id db_id schema name description] :as tbl} tables]+ (let [target-db-id (in-db->target db_id)+ existing-id (when target-db-id (target-lookup [target-db-id schema name]))]+ (cond+ (nil? target-db-id)+ (swap! state update-in [:tables :missing] conj+ (cond-> {:db_id db_id :name name}+ (some? schema) (assoc :schema schema)))++ existing-id+ (do+ (when (some? description)+ (t2/update! :model/Table existing-id {:description description}))+ (swap! state #(-> %+ (update-in [:tables :matched] inc)+ (assoc-in [:in-tbl->target id] existing-id))))++ :else+ (vswap! to-insert conj [id (new-table-row target-db-id tbl)]))))+ (when-some [rows (seq @to-insert)]+ (let [new-ids (t2/insert-returning-pks! :model/Table (mapv second rows))]+ (swap! state (fn [s]+ (reduce (fn [s [[incoming-id _] new-id]]+ (-> s+ (update-in [:tables :created] inc)+ (assoc-in [:in-tbl->target incoming-id] new-id)))+ s+ (map vector rows new-ids))))))))++(defn- import-fields!+ "Phase 2. Match incoming fields against the target by full (table, parent-path, name)+ using `build-target-field-pathmap`; UPDATE matched rows with editable metadata (minus+ parent_id / fk_target_field_id), bulk INSERT the rest with parent_id left NULL."+ [state fields incoming-by-id in-tbl->target]+ (let [path-lookup (build-target-field-pathmap (vals in-tbl->target))+ to-insert (volatile! [])]+ (doseq [{:keys [id table_id] :as fld} fields]+ (let [target-tbl (in-tbl->target table_id)+ path (when target-tbl (incoming-field-path incoming-by-id id))+ existing-id (when target-tbl (path-lookup [target-tbl path]))]+ (cond+ (nil? target-tbl)+ (swap! state update-in [:fields :missing] conj+ {:table_id table_id+ :path (incoming-field-path incoming-by-id id)})++ existing-id+ (let [patch (matched-field-patch fld)]+ (when (seq patch)+ (t2/update! :model/Field existing-id patch))+ (swap! state #(-> %+ (update-in [:fields :matched] inc)+ (assoc-in [:in-fld->target id] existing-id))))++ :else+ (vswap! to-insert conj [id (new-field-row target-tbl fld)]))))+ (when-some [rows (seq @to-insert)]+ (let [new-ids (t2/insert-returning-pks! :model/Field (mapv second rows))]+ (swap! state (fn [s]+ (reduce (fn [s [[incoming-id _] new-id]]+ (-> s+ (update-in [:fields :created] inc)+ (assoc-in [:in-fld->target incoming-id] new-id)+ (update :created-fld-ids conj incoming-id)))+ s+ (map vector rows new-ids))))))))++(defn- resolve-field-references!+ "Phase 3. Link parent_id and fk_target_field_id now that every resolvable field has a+ target id. parent_id is only set on rows we just created — matched rows keep their+ existing parent so we never restructure a nested field in place. fk_target_field_id is+ user-editable and is updated for both matched and created rows."+ [fields in-fld->target created-fld-ids]+ (doseq [{:keys [id parent_id fk_target_field_id]} fields]+ (let [tid (in-fld->target id)+ patch (cond-> {}+ (and (contains? created-fld-ids id) (some? parent_id))+ (assoc :parent_id (in-fld->target parent_id))++ (some? fk_target_field_id)+ (assoc :fk_target_field_id (in-fld->target fk_target_field_id)))]+ (when (and tid (seq patch))+ (t2/update! :model/Field tid patch)))))++(defn- import-metadata!*+ "Core logic for POST /metadata. See the endpoint doc for behavior."+ [{:keys [databases tables fields]}]+ (let [db-by-key (build-db-lookup)+ in-db->target (into {}+ (keep (fn [{:keys [id name engine]}]+ (when-some [tid (db-by-key [name (engine-name engine)])]+ [id tid])))+ databases)+ missing-dbs (mapv #(select-keys % [:name :engine])+ (remove #(in-db->target (:id %)) databases))+ incoming-by-id (m/index-by :id fields)+ state (atom {:tables {:matched 0 :created 0 :missing []}+ :fields {:matched 0 :created 0 :missing []}+ :in-tbl->target {}+ :in-fld->target {}+ :created-fld-ids #{}})]+ (t2/with-transaction [_conn]+ (import-tables! state tables in-db->target)+ (import-fields! state fields incoming-by-id (:in-tbl->target @state))+ (let [{:keys [in-fld->target created-fld-ids]} @state]+ (resolve-field-references! fields in-fld->target created-fld-ids)))+ (let [{:keys [tables fields]} @state]+ {:databases {:matched (count in-db->target) :missing missing-dbs}+ :tables tables+ :fields fields})))++(mr/def ::metadata-import-report+ [:map+ [:databases [:map [:matched :int] [:missing [:sequential :map]]]]+ [:tables [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]+ [:fields [:map [:matched :int] [:created :int] [:missing [:sequential :map]]]]])++(api.macros/defendpoint :post "/metadata"+ :- ::metadata-import-report+ "Import database/table/field metadata previously exported from `GET /api/database/metadata`.++ Entities are matched by natural key — databases by `(name, engine)`, tables by+ `(database, schema, name)`, fields by `(table, parent-path, name)` — so the numeric ids+ in the payload are only used to link fields to their tables within the request.++ Tables and fields that don't exist on the target are created when their parent+ (database for tables; table for fields) is present on the target. Databases are+ not auto-created — missing databases are reported instead. Field `fk_target_field_id`+ is re-resolved through the natural-key lookup after all fields exist.++ For matched entities, only user-editable metadata is written: table `description`;+ field `description`, `semantic_type`, `coercion_strategy`, `effective_type`, and+ `fk_target_field_id`. For newly-created fields, `base_type` and `database_type` are+ also populated from the payload. Keys absent from the payload (including null values)+ are left untouched on matched entities.++ Returns counts of matched + created entities per type plus a list of entities in the+ payload that could not be placed (their parent was missing on the target)."+ [_route-params+ _query-params+ body :- ::databases-metadata-response]+ (api/check-superuser)+ (import-metadata!* body))+;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------;; Since the normal `:id` param in the normal version of the endpoint will never match with negative numbersdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 105d291ffe4..f0f283f6f96 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -804,6 +804,130 @@(is (= {:databases [] :tables [] :fields []}(mt/user-http-request :rasta :get 202 "database/metadata")))))))+(deftest databases-metadata-import-test+ (testing "POST /api/database/metadata"+ (mt/with-temp [:model/Database {db-id :id} {:name "import-db" :engine :h2}+ :model/Table {t-id :id} {:db_id db-id :name "orders" :schema "PUBLIC"+ :description "original"}+ :model/Field {pk-id :id} {:table_id t-id :name "id" :base_type :type/Integer+ :database_type "BIGINT"}+ :model/Field {fk-id :id} {:table_id t-id :name "order_id" :base_type :type/Integer+ :database_type "BIGINT"}]+ (testing "matched entities are updated; missing tables/fields are created when parent exists"+ ;; Payload carries ids from "another" instance — here we reuse our own ids, but the+ ;; endpoint matches by natural key regardless of what the numeric ids are.+ (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}+ {:id 9999 :name "does-not-exist" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"+ :description "updated via import"}+ {:id 9998 :db_id db-id :name "new_table" :schema "PUBLIC"+ :description "created via import"}]+ :fields [{:id pk-id :table_id t-id :name "id"+ :base_type "type/Integer" :database_type "BIGINT"+ :semantic_type "type/PK"+ :description "primary key"}+ {:id fk-id :table_id t-id :name "order_id"+ :base_type "type/Integer" :database_type "BIGINT"+ :semantic_type "type/FK"+ :fk_target_field_id pk-id}+ {:id 9997 :table_id t-id :name "new_field"+ :base_type "type/Integer" :database_type "INT"+ :description "created via import"+ :semantic_type "type/Quantity"}+ {:id 9996 :table_id 9998 :name "new_table_field"+ :base_type "type/Text" :database_type "VARCHAR"}]}+ report (mt/user-http-request :crowberto :post 200+ "database/metadata" payload)]+ (is (=? {:databases {:matched 1 :missing [{:name "does-not-exist"}]}+ :tables {:matched 1 :created 1 :missing []}+ :fields {:matched 2 :created 2 :missing []}}+ report))+ (is (= "updated via import" (t2/select-one-fn :description :model/Table :id t-id)))+ (is (= :type/PK (t2/select-one-fn :semantic_type :model/Field :id pk-id)))+ (is (= "primary key" (t2/select-one-fn :description :model/Field :id pk-id)))+ (is (= pk-id (t2/select-one-fn :fk_target_field_id :model/Field :id fk-id)))+ (testing "new table was created under the matched database"+ (let [new-tbl (t2/select-one :model/Table :db_id db-id :name "new_table")]+ (is (some? new-tbl))+ (is (= "created via import" (:description new-tbl)))+ (is (true? (:active new-tbl)))))+ (testing "new field was created under the existing table"+ (let [new-fld (t2/select-one :model/Field :table_id t-id :name "new_field")]+ (is (some? new-fld))+ (is (= :type/Integer (:base_type new-fld)))+ (is (= "INT" (:database_type new-fld)))+ (is (= "created via import" (:description new-fld)))+ (is (= :type/Quantity (:semantic_type new-fld)))))+ (testing "new field was created under a newly-created table"+ (let [new-tbl-id (t2/select-one-pk :model/Table :db_id db-id :name "new_table")]+ (is (some? (t2/select-one :model/Field :table_id new-tbl-id :name "new_table_field")))))))++ (testing "fields whose database is missing on the target are reported as missing"+ (let [payload {:databases [{:id 9999 :name "does-not-exist" :engine "h2"}]+ :tables [{:id 9998 :db_id 9999 :name "x" :schema "PUBLIC"}]+ :fields [{:id 9997 :table_id 9998 :name "y" :base_type "type/Integer"}]}+ report (mt/user-http-request :crowberto :post 200+ "database/metadata" payload)]+ (is (=? {:tables {:matched 0 :created 0 :missing [{:name "x"}]}+ :fields {:matched 0 :created 0 :missing [{:path ["y"]}]}}+ report))))++ (testing "base_type and database_type are never overwritten"+ (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]+ :fields [{:id pk-id :table_id t-id :name "id"+ :base_type "type/Text" :database_type "TEXT"+ :description "still a pk"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (is (= :type/Integer (t2/select-one-fn :base_type :model/Field :id pk-id)))+ (is (= "BIGINT" (t2/select-one-fn :database_type :model/Field :id pk-id)))))++ (testing "nested fields are matched by parent path"+ (mt/with-temp [:model/Field {parent-id :id} {:table_id t-id :name "payload"+ :base_type :type/JSON :database_type "JSON"}+ :model/Field {child-id :id} {:table_id t-id :parent_id parent-id :name "amount"+ :base_type :type/Integer :database_type "BIGINT"}]+ (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]+ :fields [{:id 1 :table_id t-id :name "payload" :base_type "type/JSON"}+ {:id 2 :table_id t-id :parent_id 1 :name "amount"+ :base_type "type/Integer"+ :description "nested description"+ :semantic_type "type/Quantity"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (is (= "nested description" (t2/select-one-fn :description :model/Field :id child-id)))+ (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id child-id))))))++ (testing "fields with the same leaf name at different parent paths are matched independently"+ ;; A root-level `amount` and a nested `payload.amount` coexist on the same table.+ ;; Matching by full parent path must update each without clobbering the other.+ (mt/with-temp [:model/Field {root-amount-id :id} {:table_id t-id :name "amount"+ :base_type :type/Integer :database_type "BIGINT"}+ :model/Field {parent-id :id} {:table_id t-id :name "payload"+ :base_type :type/JSON :database_type "JSON"}+ :model/Field {nested-amount-id :id} {:table_id t-id :parent_id parent-id :name "amount"+ :base_type :type/Integer :database_type "BIGINT"}]+ (let [payload {:databases [{:id db-id :name "import-db" :engine "h2"}]+ :tables [{:id t-id :db_id db-id :name "orders" :schema "PUBLIC"}]+ :fields [{:id 10 :table_id t-id :name "amount"+ :base_type "type/Integer"+ :description "root amount"+ :semantic_type "type/Quantity"}+ {:id 11 :table_id t-id :name "payload" :base_type "type/JSON"}+ {:id 12 :table_id t-id :parent_id 11 :name "amount"+ :base_type "type/Integer"+ :description "nested amount"+ :semantic_type "type/Currency"}]}]+ (mt/user-http-request :crowberto :post 200 "database/metadata" payload)+ (is (= "root amount" (t2/select-one-fn :description :model/Field :id root-amount-id)))+ (is (= :type/Quantity (t2/select-one-fn :semantic_type :model/Field :id root-amount-id)))+ (is (= "nested amount" (t2/select-one-fn :description :model/Field :id nested-amount-id)))+ (is (= :type/Currency (t2/select-one-fn :semantic_type :model/Field :id nested-amount-id))))))++ (testing "non-superusers are rejected"+ (mt/user-http-request :rasta :post 403 "database/metadata"+ {:databases [] :tables [] :fields []})))))+(deftest ^:parallel fetch-database-metadata-test(testing "GET /api/database/:id/metadata"(is (= (merge (dissoc (db-details) :details :write_data_details :router_user_attribute)