The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Set nfc_path for mongo during sync (#73767)
Alexander Polyankin24054cb45848
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex b5e236df211..357e5908a93 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -371,16 +371,24 @@(defn- ftree->nested-fields"Create nested-fields structure suitable for :fields key of structure return by [[driver/describe-table]]. `ftree`- is a tree that represents sampled mongo documents. See the [[dbfields->ftree]] for details."+ is a tree that represents sampled mongo documents. See the [[dbfields->ftree]] for details.++ Each nested field is annotated with `:nfc-path` — the chain of names from the root document down to and+ including the field's own name (matching the convention used by the sql-jdbc nested json paths)."[ftree]- (letfn [(ftree->nested-fields*- [ftree*]- (-> ftree*- (cond-> (contains? ftree* :children)- (-> (update :children #(set (map ftree->nested-fields* (vals %))))- (set/rename-keys {:children :nested-fields})))- (dissoc :index)))]- (:nested-fields (ftree->nested-fields* ftree))))+ (letfn [(walk+ [field ancestor-path]+ (let [self-path (conj ancestor-path (:name field))]+ (cond-> field+ (seq ancestor-path)+ (assoc :nfc-path self-path)+ (contains? field :children)+ (-> (update :children+ (fn [children]+ (set (map #(walk % self-path) (vals children)))))+ (set/rename-keys {:children :nested-fields}))+ true (dissoc :index))))]+ (set (map #(walk % []) (vals (:children ftree))))))(defn- fetch-dbfields-rff[_metadata]diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex 9375ee271b2..5b9eec2e412 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -156,12 +156,19 @@{:arglists '([field])}driver-api/dispatch-by-clause-name-or-class)-(defn- col->name-components [{:keys [parent-id], field-name :name, :as _col}]- (concat- ;; TODO (Cam 8/11/25) -- this should be using `:nfc-path` instead of looking this up the hard way- (when parent-id- (col->name-components (driver-api/field (driver-api/metadata-provider) parent-id)))- [field-name]))+(defn- col->name-components [{:keys [parent-id nfc-path], field-name :name, :as _col}]+ (cond+ ;; mongo sync stores `:nfc-path` with the field's own name as the last element, matching sql-jdbc nested json+ (seq nfc-path)+ nfc-path++ ;; fall back to walking `:parent-id` for fields synced before `:nfc-path` was populated+ parent-id+ (concat (col->name-components (driver-api/field (driver-api/metadata-provider) parent-id))+ [field-name])++ :else+ [field-name]))(mu/defn field->name"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."@@ -1513,6 +1520,21 @@ function(bin) {;;; ---------------------------------------------------- order-by ----------------------------------------------------+(defn- field-id->path+ "Return the full document-path components for `field-id` as a vector of strings. Uses [[col->name-components]],+ which prefers `:nfc-path` and falls back to walking `:parent-id` for fields synced before `:nfc-path` was populated."+ [field-id]+ (vec (col->name-components (driver-api/field (driver-api/metadata-provider) field-id))))++(defn- field-clauses->id->path+ "Build a map of `field-id -> path-vector` for all `:field` clauses in `fields` that reference an integer ID."+ [fields]+ (into {}+ (keep (fn [[agg-type field-id & _]]+ (when (and (= agg-type :field) (integer? field-id))+ [field-id (field-id->path field-id)])))+ fields))+(defn- remove-parent-fields"Removes any and all entries in `fields` that are parents of another field in `fields`. This is necessary because asof MongoDB 4.4, including both will result in an error (see:@@ -1520,18 +1542,15 @@ function(bin) {Removing parents is useful when sorting, because leaf fields sort."[fields]- (let [parent->child-id (reduce (fn [acc [agg-type field-id & _]]- (if (and (= agg-type :field)- (integer? field-id))- (let [{:keys [parent-id], :as field} (driver-api/field (driver-api/metadata-provider) field-id)]- (if parent-id- (update acc parent-id conj (u/the-id field))- acc))- acc))- {}- fields)]+ (let [id->path (field-clauses->id->path fields)+ parent-paths (into #{}+ (keep (fn [path]+ (when (> (count path) 1)+ (vec (butlast path)))))+ (vals id->path))](remove (fn [[_ field-id & _]]- (and (integer? field-id) (contains? parent->child-id field-id)))+ (and (integer? field-id)+ (contains? parent-paths (id->path field-id))))fields)))(defn- remove-child-fields@@ -1542,17 +1561,13 @@ function(bin) {Removing children is useful when projecting, because the return value of a mongo query is json, and so a parentincludes all of its children."[fields]- (let [field-ids (into #{}- (map (fn [[agg-type field-id]]- (when (and (= agg-type :field)- (integer? field-id))- field-id)))- fields)]+ (let [id->path (field-clauses->id->path fields)+ all-paths (set (vals id->path))](remove (fn [[agg-type field-id]]- (when (and (= agg-type :field)- (integer? field-id))- (let [{:keys [parent-id]} (driver-api/field (driver-api/metadata-provider) field-id)]- (and parent-id (contains? field-ids parent-id)))))+ (when (and (= agg-type :field) (integer? field-id))+ (let [path (id->path field-id)]+ (and (> (count path) 1)+ (contains? all-paths (vec (butlast path)))))))fields)))(defn- handle-order-by [{:keys [order-by breakout aggregation]} pipeline-ctx]diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.cljindex d6cf8776cf4..303f65ad786 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj@@ -270,10 +270,12 @@{:name "mixed_uuid", :database-type "binData", :base-type :type/MongoBinData, :database-position 7}{:name "mixed_not_uuid", :database-type "binData", :base-type :type/MongoBinData, :database-position 6}{:name "nested_mixed_uuid", :database-type "object", :base-type :type/Dictionary, :database-position 8,- :nested-fields #{{:name "nested_data", :database-type "binData", :base-type :type/MongoBinData, :database-position 9}}+ :nested-fields #{{:name "nested_data", :database-type "binData", :base-type :type/MongoBinData, :database-position 9,+ :nfc-path ["nested_mixed_uuid" "nested_data"]}}:visibility-type :details-only}{:name "nested_mixed_not_uuid", :database-type "object", :base-type :type/Dictionary, :database-position 4,- :nested-fields #{{:name "nested_data_2", :database-type "binData", :base-type :type/MongoBinData, :database-position 5}}+ :nested-fields #{{:name "nested_data_2", :database-type "binData", :base-type :type/MongoBinData, :database-position 5,+ :nfc-path ["nested_mixed_not_uuid" "nested_data_2"]}}:visibility-type :details-only}}}(driver/describe-table :mongo (mt/db) (t2/select-one :model/Table :id (mt/id :nested-bindata)))))))))@@ -1064,7 +1066,8 @@:database-position 2,:base-type :type/Dictionary,:name "b",- :nested-fields #{{:database-type "int", :database-position 3, :base-type :type/Integer, :name "c"}}}}}}+ :nfc-path ["a" "b"],+ :nested-fields #{{:database-type "int", :database-position 3, :base-type :type/Integer, :name "c", :nfc-path ["a" "b" "c"]}}}}}}@nested-fields)))))(deftest nulls-are-last-test@@ -1082,7 +1085,7 @@:database-position 1,:base-type :type/Dictionary,:name "a",- :nested-fields #{{:database-type "string", :database-position 2, :base-type :type/Text, :name "b"}}}+ :nested-fields #{{:database-type "string", :database-position 2, :base-type :type/Text, :name "b", :nfc-path ["a" "b"]}}}{:database-type "objectId", :database-position 0, :base-type :type/MongoBSONID, :name "_id", :pk? true}}@nested-fields)))))@@ -1102,7 +1105,7 @@:database-position 1,:base-type :type/Dictionary,:name "a",- :nested-fields #{{:database-type "int", :database-position 2, :base-type :type/Integer, :name "b"}}}+ :nested-fields #{{:database-type "int", :database-position 2, :base-type :type/Integer, :name "b", :nfc-path ["a" "b"]}}}{:database-type "objectId", :database-position 0, :base-type :type/MongoBSONID, :name "_id", :pk? true}}@nested-fields)))))diff --git a/src/metabase/sync/sync_metadata/fields/our_metadata.clj b/src/metabase/sync/sync_metadata/fields/our_metadata.cljindex b55ab1b20a5..4d829f3a7af 100644--- a/src/metabase/sync/sync_metadata/fields/our_metadata.clj+++ b/src/metabase/sync/sync_metadata/fields/our_metadata.clj@@ -36,7 +36,8 @@:database-partitioned (:database_partitioned field):database-required (:database_required field):visibility-type (:visibility_type field)- :preview-display (:preview_display field)}+ :preview-display (:preview_display field)+ :nfc-path (:nfc_path field)}(u/remove-nils{:pk? (:database_is_pk field):database-is-generated (:database_is_generated field)diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.cljindex 6a8f3720e40..0f377516062 100644--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj@@ -15,6 +15,12 @@[metabase.warehouse-schema.models.field-user-settings :as schema.field-user-settings][toucan2.core :as t2]))+(defn- normalize-nfc-path+ "Normalize a `nfc-path` to a vector of strings so a driver emitting keywords doesn't churn against the+ JSON-stringified values stored in the application DB."+ [path]+ (some->> path (mapv name)))+(defn- crufty-field? [db field-metadata](crufty/name? (:name field-metadata)(some-> db :settings :auto-cruft-columns)))@@ -49,7 +55,8 @@old-db-partitioned :database-partitionedold-db-required :database-requiredold-visibility-type :visibility-type- old-preview-display :preview-display} metabase-field+ old-preview-display :preview-display+ old-nfc-path :nfc-path} metabase-field{new-database-type :database-typenew-base-type :base-typenew-field-comment :field-comment@@ -61,7 +68,8 @@new-database-is-generated :database-is-generatednew-database-is-nullable :database-is-nullablenew-db-partitioned :database-partitioned- new-db-required :database-required} field-metadata+ new-db-required :database-required+ new-nfc-path :nfc-path} field-metadatanew-visibility-type (compute-new-visibility-type database field-metadata)new-database-is-auto-increment (boolean new-database-is-auto-increment)new-db-required (boolean new-db-required)@@ -98,6 +106,8 @@new-db-partitioned? (not= new-db-partitioned old-db-partitioned)new-db-required? (not= old-db-required new-db-required)new-visibility-type? (not= old-visibility-type new-visibility-type)+ new-nfc-path? (not= (normalize-nfc-path old-nfc-path)+ (normalize-nfc-path new-nfc-path));; set preview_display=false for crufty fields (prevents FieldValues from being created)is-crufty? (crufty-field? database field-metadata)set-preview-display-false? (and is-crufty? old-preview-display)@@ -205,6 +215,12 @@{:database_required new-db-required})(when new-visibility-type?{:visibility_type new-visibility-type})+ (when new-nfc-path?+ (log/infof "NFC path of %s has changed from '%s' to '%s'."+ (common/field-metadata-name-for-logging table metabase-field)+ old-nfc-path+ new-nfc-path)+ {:nfc_path new-nfc-path})(when set-preview-display-false?{:preview_display false}))];; if any updates need to be done, do them and return 1 (because 1 Field was updated), otherwise return 0diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.cljindex a0b9468f4ed..ee0b24ca637 100644--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj@@ -181,6 +181,37 @@(merge default-metadata {:database-partitioned false})(merge default-metadata {:database-partitioned true :id 1}))))))+(deftest update-nfc-path-test+ (testing "nil -> vector"+ (is (= [["Field" 1 {:nfc_path ["payload" "user"]}]]+ (updates-that-will-be-performed!+ (merge default-metadata {:nfc-path ["payload" "user"]})+ (merge default-metadata {:id 1})))))++ (testing "vector -> nil"+ (is (= [["Field" 1 {:nfc_path nil}]]+ (updates-that-will-be-performed!+ default-metadata+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))++ (testing "vector -> different vector"+ (is (= [["Field" 1 {:nfc_path ["payload" "account"]}]]+ (updates-that-will-be-performed!+ (merge default-metadata {:nfc-path ["payload" "account"]})+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))++ (testing "no change when path is identical"+ (is (= []+ (updates-that-will-be-performed!+ (merge default-metadata {:nfc-path ["payload" "user"]})+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))++ (testing "no change when driver emits keywords and DB has equivalent strings"+ (is (= []+ (updates-that-will-be-performed!+ (merge default-metadata {:nfc-path [:payload :user]})+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1}))))))+(deftest nil-database-type-test(testing (str "test that if `database-type` comes back as `nil` in the metadata from the sync process, we won't try ""to set a `nil` value in the DB -- this is against the rules -- we should set `NULL` instead. See "