Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Set nfc_path for mongo during sync (#73767)

Alexander Polyankin24054cb45848
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index b5e236df211..357e5908a93 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -371,16 +371,24 @@
(defn- ftree->nested-fields
"Create nested-fields structure suitable for :fields key of structure return by [[driver/describe-table]]. `ftree`
- is a tree that represents sampled mongo documents. See the [[dbfields->ftree]] for details."
+ is a tree that represents sampled mongo documents. See the [[dbfields->ftree]] for details.
+
+ Each nested field is annotated with `:nfc-path` — the chain of names from the root document down to and
+ including the field's own name (matching the convention used by the sql-jdbc nested json paths)."
[ftree]
- (letfn [(ftree->nested-fields*
- [ftree*]
- (-> ftree*
- (cond-> (contains? ftree* :children)
- (-> (update :children #(set (map ftree->nested-fields* (vals %))))
- (set/rename-keys {:children :nested-fields})))
- (dissoc :index)))]
- (:nested-fields (ftree->nested-fields* ftree))))
+ (letfn [(walk
+ [field ancestor-path]
+ (let [self-path (conj ancestor-path (:name field))]
+ (cond-> field
+ (seq ancestor-path)
+ (assoc :nfc-path self-path)
+ (contains? field :children)
+ (-> (update :children
+ (fn [children]
+ (set (map #(walk % self-path) (vals children)))))
+ (set/rename-keys {:children :nested-fields}))
+ true (dissoc :index))))]
+ (set (map #(walk % []) (vals (:children ftree))))))
(defn- fetch-dbfields-rff
[_metadata]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 9375ee271b2..5b9eec2e412 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -156,12 +156,19 @@
{:arglists '([field])}
driver-api/dispatch-by-clause-name-or-class)
-(defn- col->name-components [{:keys [parent-id], field-name :name, :as _col}]
- (concat
- ;; TODO (Cam 8/11/25) -- this should be using `:nfc-path` instead of looking this up the hard way
- (when parent-id
- (col->name-components (driver-api/field (driver-api/metadata-provider) parent-id)))
- [field-name]))
+(defn- col->name-components [{:keys [parent-id nfc-path], field-name :name, :as _col}]
+ (cond
+ ;; mongo sync stores `:nfc-path` with the field's own name as the last element, matching sql-jdbc nested json
+ (seq nfc-path)
+ nfc-path
+
+ ;; fall back to walking `:parent-id` for fields synced before `:nfc-path` was populated
+ parent-id
+ (concat (col->name-components (driver-api/field (driver-api/metadata-provider) parent-id))
+ [field-name])
+
+ :else
+ [field-name]))
(mu/defn field->name
"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."
@@ -1513,6 +1520,21 @@ function(bin) {
;;; ---------------------------------------------------- order-by ----------------------------------------------------
+(defn- field-id->path
+ "Return the full document-path components for `field-id` as a vector of strings. Uses [[col->name-components]],
+ which prefers `:nfc-path` and falls back to walking `:parent-id` for fields synced before `:nfc-path` was populated."
+ [field-id]
+ (vec (col->name-components (driver-api/field (driver-api/metadata-provider) field-id))))
+
+(defn- field-clauses->id->path
+ "Build a map of `field-id -> path-vector` for all `:field` clauses in `fields` that reference an integer ID."
+ [fields]
+ (into {}
+ (keep (fn [[agg-type field-id & _]]
+ (when (and (= agg-type :field) (integer? field-id))
+ [field-id (field-id->path field-id)])))
+ fields))
+
(defn- remove-parent-fields
"Removes any and all entries in `fields` that are parents of another field in `fields`. This is necessary because as
of MongoDB 4.4, including both will result in an error (see:
@@ -1520,18 +1542,15 @@ function(bin) {
Removing parents is useful when sorting, because leaf fields sort."
[fields]
- (let [parent->child-id (reduce (fn [acc [agg-type field-id & _]]
- (if (and (= agg-type :field)
- (integer? field-id))
- (let [{:keys [parent-id], :as field} (driver-api/field (driver-api/metadata-provider) field-id)]
- (if parent-id
- (update acc parent-id conj (u/the-id field))
- acc))
- acc))
- {}
- fields)]
+ (let [id->path (field-clauses->id->path fields)
+ parent-paths (into #{}
+ (keep (fn [path]
+ (when (> (count path) 1)
+ (vec (butlast path)))))
+ (vals id->path))]
(remove (fn [[_ field-id & _]]
- (and (integer? field-id) (contains? parent->child-id field-id)))
+ (and (integer? field-id)
+ (contains? parent-paths (id->path field-id))))
fields)))
(defn- remove-child-fields
@@ -1542,17 +1561,13 @@ function(bin) {
Removing children is useful when projecting, because the return value of a mongo query is json, and so a parent
includes all of its children."
[fields]
- (let [field-ids (into #{}
- (map (fn [[agg-type field-id]]
- (when (and (= agg-type :field)
- (integer? field-id))
- field-id)))
- fields)]
+ (let [id->path (field-clauses->id->path fields)
+ all-paths (set (vals id->path))]
(remove (fn [[agg-type field-id]]
- (when (and (= agg-type :field)
- (integer? field-id))
- (let [{:keys [parent-id]} (driver-api/field (driver-api/metadata-provider) field-id)]
- (and parent-id (contains? field-ids parent-id)))))
+ (when (and (= agg-type :field) (integer? field-id))
+ (let [path (id->path field-id)]
+ (and (> (count path) 1)
+ (contains? all-paths (vec (butlast path)))))))
fields)))
(defn- handle-order-by [{:keys [order-by breakout aggregation]} pipeline-ctx]
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index d6cf8776cf4..303f65ad786 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -270,10 +270,12 @@
{:name "mixed_uuid", :database-type "binData", :base-type :type/MongoBinData, :database-position 7}
{:name "mixed_not_uuid", :database-type "binData", :base-type :type/MongoBinData, :database-position 6}
{:name "nested_mixed_uuid", :database-type "object", :base-type :type/Dictionary, :database-position 8,
- :nested-fields #{{:name "nested_data", :database-type "binData", :base-type :type/MongoBinData, :database-position 9}}
+ :nested-fields #{{:name "nested_data", :database-type "binData", :base-type :type/MongoBinData, :database-position 9,
+ :nfc-path ["nested_mixed_uuid" "nested_data"]}}
:visibility-type :details-only}
{:name "nested_mixed_not_uuid", :database-type "object", :base-type :type/Dictionary, :database-position 4,
- :nested-fields #{{:name "nested_data_2", :database-type "binData", :base-type :type/MongoBinData, :database-position 5}}
+ :nested-fields #{{:name "nested_data_2", :database-type "binData", :base-type :type/MongoBinData, :database-position 5,
+ :nfc-path ["nested_mixed_not_uuid" "nested_data_2"]}}
:visibility-type :details-only}}}
(driver/describe-table :mongo (mt/db) (t2/select-one :model/Table :id (mt/id :nested-bindata)))))))))
@@ -1064,7 +1066,8 @@
:database-position 2,
:base-type :type/Dictionary,
:name "b",
- :nested-fields #{{:database-type "int", :database-position 3, :base-type :type/Integer, :name "c"}}}}}}
+ :nfc-path ["a" "b"],
+ :nested-fields #{{:database-type "int", :database-position 3, :base-type :type/Integer, :name "c", :nfc-path ["a" "b" "c"]}}}}}}
@nested-fields)))))
(deftest nulls-are-last-test
@@ -1082,7 +1085,7 @@
:database-position 1,
:base-type :type/Dictionary,
:name "a",
- :nested-fields #{{:database-type "string", :database-position 2, :base-type :type/Text, :name "b"}}}
+ :nested-fields #{{:database-type "string", :database-position 2, :base-type :type/Text, :name "b", :nfc-path ["a" "b"]}}}
{:database-type "objectId", :database-position 0, :base-type :type/MongoBSONID, :name "_id", :pk? true}}
@nested-fields)))))
@@ -1102,7 +1105,7 @@
:database-position 1,
:base-type :type/Dictionary,
:name "a",
- :nested-fields #{{:database-type "int", :database-position 2, :base-type :type/Integer, :name "b"}}}
+ :nested-fields #{{:database-type "int", :database-position 2, :base-type :type/Integer, :name "b", :nfc-path ["a" "b"]}}}
{:database-type "objectId", :database-position 0, :base-type :type/MongoBSONID, :name "_id", :pk? true}}
@nested-fields)))))
diff --git a/src/metabase/sync/sync_metadata/fields/our_metadata.clj b/src/metabase/sync/sync_metadata/fields/our_metadata.clj
index b55ab1b20a5..4d829f3a7af 100644
--- a/src/metabase/sync/sync_metadata/fields/our_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/our_metadata.clj
@@ -36,7 +36,8 @@
:database-partitioned (:database_partitioned field)
:database-required (:database_required field)
:visibility-type (:visibility_type field)
- :preview-display (:preview_display field)}
+ :preview-display (:preview_display field)
+ :nfc-path (:nfc_path field)}
(u/remove-nils
{:pk? (:database_is_pk field)
:database-is-generated (:database_is_generated field)
diff --git a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
index 6a8f3720e40..0f377516062 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_metadata.clj
@@ -15,6 +15,12 @@
[metabase.warehouse-schema.models.field-user-settings :as schema.field-user-settings]
[toucan2.core :as t2]))
+(defn- normalize-nfc-path
+ "Normalize a `nfc-path` to a vector of strings so a driver emitting keywords doesn't churn against the
+ JSON-stringified values stored in the application DB."
+ [path]
+ (some->> path (mapv name)))
+
(defn- crufty-field? [db field-metadata]
(crufty/name? (:name field-metadata)
(some-> db :settings :auto-cruft-columns)))
@@ -49,7 +55,8 @@
old-db-partitioned :database-partitioned
old-db-required :database-required
old-visibility-type :visibility-type
- old-preview-display :preview-display} metabase-field
+ old-preview-display :preview-display
+ old-nfc-path :nfc-path} metabase-field
{new-database-type :database-type
new-base-type :base-type
new-field-comment :field-comment
@@ -61,7 +68,8 @@
new-database-is-generated :database-is-generated
new-database-is-nullable :database-is-nullable
new-db-partitioned :database-partitioned
- new-db-required :database-required} field-metadata
+ new-db-required :database-required
+ new-nfc-path :nfc-path} field-metadata
new-visibility-type (compute-new-visibility-type database field-metadata)
new-database-is-auto-increment (boolean new-database-is-auto-increment)
new-db-required (boolean new-db-required)
@@ -98,6 +106,8 @@
new-db-partitioned? (not= new-db-partitioned old-db-partitioned)
new-db-required? (not= old-db-required new-db-required)
new-visibility-type? (not= old-visibility-type new-visibility-type)
+ new-nfc-path? (not= (normalize-nfc-path old-nfc-path)
+ (normalize-nfc-path new-nfc-path))
;; set preview_display=false for crufty fields (prevents FieldValues from being created)
is-crufty? (crufty-field? database field-metadata)
set-preview-display-false? (and is-crufty? old-preview-display)
@@ -205,6 +215,12 @@
{:database_required new-db-required})
(when new-visibility-type?
{:visibility_type new-visibility-type})
+ (when new-nfc-path?
+ (log/infof "NFC path of %s has changed from '%s' to '%s'."
+ (common/field-metadata-name-for-logging table metabase-field)
+ old-nfc-path
+ new-nfc-path)
+ {:nfc_path new-nfc-path})
(when set-preview-display-false?
{:preview_display false}))]
;; if any updates need to be done, do them and return 1 (because 1 Field was updated), otherwise return 0
diff --git a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
index a0b9468f4ed..ee0b24ca637 100644
--- a/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/sync_metadata_test.clj
@@ -181,6 +181,37 @@
(merge default-metadata {:database-partitioned false})
(merge default-metadata {:database-partitioned true :id 1}))))))
+(deftest update-nfc-path-test
+ (testing "nil -> vector"
+ (is (= [["Field" 1 {:nfc_path ["payload" "user"]}]]
+ (updates-that-will-be-performed!
+ (merge default-metadata {:nfc-path ["payload" "user"]})
+ (merge default-metadata {:id 1})))))
+
+ (testing "vector -> nil"
+ (is (= [["Field" 1 {:nfc_path nil}]]
+ (updates-that-will-be-performed!
+ default-metadata
+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))
+
+ (testing "vector -> different vector"
+ (is (= [["Field" 1 {:nfc_path ["payload" "account"]}]]
+ (updates-that-will-be-performed!
+ (merge default-metadata {:nfc-path ["payload" "account"]})
+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))
+
+ (testing "no change when path is identical"
+ (is (= []
+ (updates-that-will-be-performed!
+ (merge default-metadata {:nfc-path ["payload" "user"]})
+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1})))))
+
+ (testing "no change when driver emits keywords and DB has equivalent strings"
+ (is (= []
+ (updates-that-will-be-performed!
+ (merge default-metadata {:nfc-path [:payload :user]})
+ (merge default-metadata {:nfc-path ["payload" "user"] :id 1}))))))
+
(deftest nil-database-type-test
(testing (str "test that if `database-type` comes back as `nil` in the metadata from the sync process, we won't try "
"to set a `nil` value in the DB -- this is against the rules -- we should set `NULL` instead. See "