Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fk deser completeness test (#74502)

Nicola Momettod2f4caa79a5d
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 03ab5e8667c..69af0a81698 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -2018,8 +2018,9 @@
(is (not (contains? ser :id)))
(is (not (contains? ser :use_verified_content)))
- (testing "metabot depends on its model entities"
- (is (= #{[{:model "Card" :id model-eid}]}
+ (testing "metabot depends on its model entities and collection"
+ (is (= #{[{:model "Card" :id model-eid}]
+ [{:model "Collection" :id coll-eid}]}
(set (serdes/dependencies ser)))))
(testing "metabot storage-path uses top-level metabots directory"
@@ -2069,8 +2070,9 @@
(is (not (contains? ser :id)))
(is (not (contains? ser :use_verified_content)))
- (testing "metabot depends on its prompts' cards"
- (is (= #{[{:model "Card" :id card-eid}]}
+ (testing "metabot depends on its prompts' cards and its collection"
+ (is (= #{[{:model "Card" :id card-eid}]
+ [{:model "Collection" :id coll-eid}]}
(set (serdes/dependencies ser))))))))))
(deftest document-test
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
index 8e140af3b11..d160e35a34b 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
@@ -142,3 +142,72 @@
(doseq [field-kw (keys declared-defaults)]
(testing (format "`%s.%s` in :defaults is a serialized field" m (name field-kw))
(is (contains? serialized-fields field-kw)))))))))))
+
+(def ^:private auto-synthesized-fk-targets
+ "FK target models whose importers auto-create a stub entity if the target is missing, so
+ they don't have to be declared in `serdes/dependencies`. Everything else MUST be declared
+ — otherwise `*import-fk*` throws \"Could not find foreign key target\" and aborts the
+ whole import (see GDGT-2444 for an example)."
+ #{:model/User :model/Table :model/Field})
+
+(def ^:private fk-completeness-known-exceptions
+ "`[model field]` pairs that look like FK-completeness violations but are intentionally
+ exempt. The map value is a human reason kept for grep-ability."
+ {["Database" :router_database_id]
+ "router_database_id rows are filtered out of `extract-query` (router DBs aren't serialized), so this FK never reaches the load path."})
+
+(defn- inlined-via-nested?
+ "True if `model-name` is loaded only as a nested child of some parent, never as a root.
+ Such models override `serdes/generate-path` to return nil so the extract pipeline skips
+ them as top-level entities, and their FKs are handled by the parent's `dependencies`."
+ [model-name]
+ (nil? (serdes/generate-path model-name {:entity_id "probe" :name "probe"})))
+
+(defn- direct-fks
+ "Return a seq of `[field target-model]` for every entry in `transform-spec` that is a
+ plain `(serdes/fk ...)` declaration whose target needs a `dependencies` entry. Nested
+ child specs are NOT walked — their FKs flow through the parent's bespoke `dependencies`
+ method which knows how to dig into the nested data, and verifying that takes a
+ parent-by-parent treatment that's out of scope here."
+ [transform-spec]
+ (for [[field transform] transform-spec
+ :when (and (map? transform)
+ (::serdes/fk transform)
+ (not (::serdes/nested transform)))
+ :let [target (::serdes/fk-model transform)]
+ :when (and target
+ (not (auto-synthesized-fk-targets target)))]
+ [field target]))
+
+(deftest ^:parallel serialization-direct-fk-dependencies-completeness-test
+ (testing (str "Every direct FK declared via `(serdes/fk ...)` in a loadable (non-inlined) "
+ "model's `make-spec` is surfaced in that model's `serdes/dependencies` — "
+ "otherwise `*import-fk*` will throw \"Could not find foreign key target\" at "
+ "import time (see GDGT-2444).")
+ (let [inlined? (set serdes.models/inlined-models)]
+ (doseq [m (-> (methods serdes/make-spec) (dissoc :default) keys)
+ ;; Inlined children are loaded inside their parent, so the FK-resolution
+ ;; check applies to the parent's `dependencies`, not theirs. Skipped either
+ ;; via the explicit list in `serdes.models/inlined-models` or by detecting a
+ ;; nil `generate-path` (the convention for nested-only models like
+ ;; QueryAction / HTTPAction / ImplicitAction).
+ :when (not (or (inlined? m) (inlined-via-nested? m)))
+ :let [fks (->> (:transform (serdes/make-spec m nil))
+ direct-fks
+ (remove (fn [[field _]]
+ (contains? fk-completeness-known-exceptions [m field]))))]
+ :when (seq fks)]
+ (testing (format "%s\n" m)
+ (let [entity (into {:serdes/meta [{:model m :id "self"}]}
+ (for [[field _] fks]
+ [field (str "fake-eid-" (name field))]))
+ deps (set (serdes/dependencies entity))
+ dep-models (set (keep (comp :model peek) deps))]
+ (doseq [[field target] fks]
+ (testing (format "FK `%s` -> `%s` must appear in `(serdes/dependencies)`"
+ field target)
+ (is (contains? dep-models (name target))
+ (format (str "`(serdes/dependencies %s)` did not include a `%s` entry "
+ "when `%s` was populated. Add the FK to the model's "
+ "`dependencies` method (see Table/Transform for an example).")
+ m (name target) field))))))))))
diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.clj
index 97165dbc945..b79e1af444e 100644
--- a/src/metabase/metabot/models/metabot.clj
+++ b/src/metabase/metabot/models/metabot.clj
@@ -35,8 +35,9 @@
[:name])
(defmethod serdes/dependencies "Metabot"
- [{:keys [prompts]}]
- (set (mapcat serdes/dependencies prompts)))
+ [{:keys [collection_id prompts]}]
+ (cond-> (set (mapcat serdes/dependencies prompts))
+ collection_id (conj [{:model "Collection" :id collection_id}])))
(defmethod serdes/generate-path "Metabot" [_ metabot]
[(serdes/infer-self-path "Metabot" metabot)])
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 334e22dbc51..93616711039 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -1868,16 +1868,19 @@
;;; Common transformers
(defn fk "Export Foreign Key" [model & [field-name]]
- (cond
- ;; this `::fk` is used in tests to determine that foreign keys are handled
- (= model :model/User) {::fk true :export *export-user* :import *import-user*}
- (= model :model/Database) {::fk true :export *export-database-fk* :import *import-database-fk*}
- (= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}
- (= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}
- field-name {::fk true
- :export #(*export-fk-keyed* % model field-name)
- :import #(*import-fk-keyed* % model field-name)}
- :else {::fk true :export #(*export-fk* % model) :import #(*import-fk* % model)}))
+ (let [base {::fk true ::fk-model model}]
+ (cond
+ ;; this `::fk` is used in tests to determine that foreign keys are handled
+ (= model :model/User) (assoc base :export *export-user* :import *import-user*)
+ (= model :model/Database) (assoc base :export *export-database-fk* :import *import-database-fk*)
+ (= model :model/Table) (assoc base :export *export-table-fk* :import *import-table-fk*)
+ (= model :model/Field) (assoc base :export *export-field-fk* :import *import-field-fk*)
+ field-name (assoc base
+ :export #(*export-fk-keyed* % model field-name)
+ :import #(*import-fk-keyed* % model field-name))
+ :else (assoc base
+ :export #(*export-fk* % model)
+ :import #(*import-fk* % model)))))
(defn nested "Nested entities" [model backward-fk opts]
(let [model-name (name model)