Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Validate serialization dependencies before export (GHY-4010) (#76763)

Eric Normandff5e3b6973ce
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
index 75b952dbc0d..6a6d3d14445 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
@@ -116,7 +116,7 @@
(try
(zipmap (or (get @dep-cache dep)
(get (swap! dep-cache assoc dep
- (serdes/dependencies
+ (serdes/deserialization-dependencies
(serialization/ingest-one ingestable dep)))
dep))
(repeat dep))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj
new file mode 100644
index 00000000000..facc4d9f1a5
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj
@@ -0,0 +1,133 @@
+(ns metabase-enterprise.serialization.v2.dependency-validation
+ "Export-time dependency validation for serialization.
+
+ Before an export is written, every entity that will be extracted is checked to make sure all of its references are
+ satisfiable in the resulting archive: content references (Cards, Dashboards, …) must themselves be part of the
+ export, and data-model references (Database, Table, Field) must exist in the source appdb. An unsatisfied reference
+ would otherwise become a dangling reference or a malformed portable id in the archive, breaking the import.
+
+ Historically this was the collection-id, Card-only \"escape analysis\"; it is now a general dependency-satisfaction
+ check driven by [[metabase.models.serialization/serialization-dependencies]]."
+ (:require
+ [clojure.string :as str]
+ [medley.core :as m]
+ [metabase-enterprise.serialization.v2.models :as serdes.models]
+ [metabase.models.serialization :as serdes]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- collection-label [coll-id]
+ (if coll-id
+ (let [collection (t2/hydrate (t2/select-one :model/Collection :id coll-id) :ancestors)
+ names (->> (conj (:ancestors collection) collection)
+ (map :name)
+ (str/join " > "))]
+ (format "%d: %s" coll-id names))
+ "[no collection]"))
+
+(defn- entity-label [{:keys [model id]}]
+ (let [entity (t2/select-one [model :collection_id :name] :id id)]
+ (format "%s %d (%s from collection %s)" (name model) id (:name entity) (collection-label (:collection_id entity)))))
+
+(defn- resize-batch
+ [[model batch]]
+ (for [batch (partition-all serdes/query-batch-size batch)]
+ [model batch]))
+
+(defn- entity-deps
+ "The dependency contribution of a single entity.
+
+ Returns a `{:visited :deps}`. `serialization-dependencies` yields `:serdes/meta` paths (each a vector whose last
+ element identifies the referenced entity), so we take that last element and tag it with the `:via` entity."
+ [entity]
+ (let [model (name (t2/model entity))
+ via [model (:id entity)]]
+ {:visited #{via}
+ :deps (into #{}
+ (map #(assoc (peek %) :via via))
+ (serdes/serialization-dependencies model entity))}))
+
+(defn- merge-entity-deps
+ "Monoid merging [[entity-deps]] contributions: unions the `:visited` and `:deps` sets. Identity is empty sets."
+ ([] {:visited #{} :deps #{}})
+ ([acc] acc)
+ ([acc contribution] (merge-with into acc contribution)))
+
+(defn- collect-dependencies
+ "Take the model/ids and return {:visited :deps}. Entities are streamed in batches, not held in memory."
+ [by-model coll-set opts]
+ (let [content-models (set serdes.models/content)]
+ (transduce
+ (comp
+ (filter (comp content-models key))
+ (mapcat resize-batch)
+ (mapcat (fn [[model batch]]
+ (serdes/extract-query model (merge opts {:collection-set coll-set
+ :where [:in :id batch]}))))
+ (map entity-deps))
+ merge-entity-deps
+ by-model)))
+
+(defn- existing-ids
+ "The subset of `ids` that exist as rows of `model` (a model-name string), queried in bounded `:in` batches."
+ [model ids]
+ (into #{}
+ (comp (distinct)
+ (partition-all serdes/query-batch-size)
+ (mapcat #(t2/select-pks-set (keyword "model" model) {:where [:in :id %]})))
+ ids))
+
+(def ^:private structural-content-models
+ "Content models whose absence from the archive is tolerated on import, so a reference to one is never a completeness
+ failure. A selective export routinely omits the Collection tree an entity lives in; the entity then loads at the
+ root of the target rather than producing a dangling reference."
+ #{"Collection"})
+
+(defn- unsatisfied-dependencies
+ "The `deps` that won't be satisfied in the archive, each tagged with a `:reason` (see the namespace docstring for the
+ data-model vs content classification). Two subtleties are encoded below: a content reference to a *deleted* entity
+ is not a failure (export can't emit a portable id for a gone row, so `fk-elide` just drops it), and
+ [[structural-content-models]] references are ignored entirely."
+ [deps visited analytics-cards]
+ (let [content-models (set serdes.models/content)
+ {content-deps true data-deps false} (group-by #(contains? content-models (:model %)) deps)
+ content-deps (remove (comp structural-content-models :model) content-deps)
+ existing-data (m/map-kv-vals existing-ids (u/group-by :model :id data-deps))
+ existing-content (m/map-kv-vals existing-ids (u/group-by :model :id content-deps))]
+ (concat
+ (for [{:keys [model id] :as dep} content-deps
+ :when (and (contains? (get existing-content model) id)
+ (not (contains? visited [model id]))
+ (not (and (= model "Card") (contains? analytics-cards id))))]
+ (assoc dep :reason :not-in-export))
+ (for [{:keys [model id] :as dep} data-deps
+ :when (not (contains? (get existing-data model) id))]
+ (assoc dep :reason :missing-in-appdb)))))
+
+(defn- unsatisfied-label [{:keys [model id via reason]}]
+ (format "%s references %s %s which %s"
+ (entity-label {:model (keyword "model" (first via)) :id (second via)})
+ model id
+ (case reason
+ :not-in-export "is not included in the export"
+ :missing-in-appdb "is missing from the source database")))
+
+(defn validate-dependencies!
+ "Validates that every entity to be extracted has all of its references satisfied (see the namespace docstring). Logs
+ a warning per unsatisfied reference and, unless `:continue-on-error`, throws to abort the export before any archive
+ is produced. `by-model` is a map of `{model-name [ids ...]}`."
+ [by-model coll-set analytics-cards opts]
+ (let [{:keys [visited deps]} (collect-dependencies by-model coll-set opts)
+ missing (unsatisfied-dependencies deps visited analytics-cards)]
+ (when (seq missing)
+ (doseq [dep missing]
+ (log/warnf "Failed to export: %s" (unsatisfied-label dep)))
+ (when-not (:continue-on-error opts)
+ (throw (ex-info (format (str "Serialization failed: %d reference(s) could not be satisfied, which would "
+ "produce an incomplete export. See the warnings above for the affected "
+ "entities. Pass continue-on-error to export anyway, skipping them.")
+ (count missing))
+ {:status-code 400}))))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 97fbe7c828e..b33d51f78d2 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -4,8 +4,8 @@
See the detailed descriptions of the (de)serialization processes in [[metabase.models.serialization]]."
(:require
[clojure.set :as set]
- [clojure.string :as str]
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
+ [metabase-enterprise.serialization.v2.dependency-validation :as dependency-validation]
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.collections.models.collection :as collection]
[metabase.models.serialization :as serdes]
@@ -71,19 +71,6 @@
(into (map :id) roots)
(into (mapcat collection/descendant-ids) roots))))
-(defn- collection-label [coll-id]
- (if coll-id
- (let [collection (t2/hydrate (t2/select-one :model/Collection :id coll-id) :ancestors)
- names (->> (conj (:ancestors collection) collection)
- (map :name)
- (str/join " > "))]
- (format "%d: %s" coll-id names))
- "[no collection]"))
-
-(defn- entity-label [{:keys [model id]}]
- (let [entity (t2/select-one [model :collection_id :name] :id id)]
- (format "%s %d (%s from collection %s)" (name model) id (:name entity) (collection-label (:collection_id entity)))))
-
(defn- parse-target [[model-name id :as target]]
(if (string? id)
(if-let [resolved-id (serdes/eid->id model-name id)]
@@ -103,68 +90,21 @@
(mapcat collection/descendant-ids)
analytics-roots)))
-(defn- escape-analysis
- "Analyzes the dependency graph to find cards that are outside the collection set (escapees).
- Returns a map with:
- - :reportable-escaped - escapees that should trigger warnings (non-analytics)
- - :analytics-card-ids - card IDs in analytics collections (should be removed from extraction but not block export)
-
- Cards that depend on analytics cards are allowed to be exported - the references will be converted
- to entity_ids during export and resolved on import since analytics cards have stable entity_ids."
- [{colls "Collection" cards "Card" :as _by-model} nodes]
- (log/tracef "Running escape analysis for %d colls and %d cards" (count colls) (count cards))
- (when-let [colls (-> colls set not-empty)]
- (let [clause {:where [:or
- [:in :collection_id colls]
- (when (contains? colls nil)
- [:= :collection_id nil])]}
- possible-pks (t2/select-pks-set :model/Card clause)
- escaped-card-ids (set/difference (set cards) possible-pks)]
- (when (seq escaped-card-ids)
- ;; Analytics cards have stable entity_ids across instances, so cards that depend on them
- ;; can still be exported: the references will be resolved on import
- (let [analytics-colls (analytics-collection-ids)
- escaped-in-analytics (if (seq analytics-colls)
- (t2/select-pks-set :model/Card {:where [:and
- [:in :id escaped-card-ids]
- [:in :collection_id analytics-colls]]})
- #{})
- reportable-escaped (set/difference escaped-card-ids escaped-in-analytics)]
- {:reportable-escaped (->> reportable-escaped
- (mapv (fn [id]
- (-> (get nodes ["Card" id])
- (assoc :escapee {:model :model/Card
- :id id})))))
- :analytics-card-ids escaped-in-analytics})))))
-
-(defn- handle-escapees!
- "Reports cards that are referenced from inside the requested collections but saved outside them.
-
- Logs a per-entity warning for each escapee.
-
- Throws if `continue-on-error?` is false."
- [escaped continue-on-error?]
- (let [dashboards (group-by #(get % "Dashboard") escaped)]
- (doseq [[dash-id escapes] (dissoc dashboards nil)]
- (log/warnf "Failed to export Dashboard %d (%s) containing Card saved outside requested collections: %s"
- dash-id
- (t2/select-one-fn :name :model/Dashboard :id dash-id)
- (str/join ", " (map #(entity-label (:escapee %)) escapes))))
- (when-let [other (not-empty (get dashboards nil))]
- (log/warnf "Failed to export Cards based on questions outside requested collections: %s"
- (str/join ", " (for [item other]
- (format "%s -> %s"
- (if (get item "Card")
- (entity-label {:model :model/Card :id (get item "Card")})
- (dissoc item :escapee))
- (entity-label (:escapee item))))))))
- (when-not continue-on-error?
- (throw (ex-info (format (str "Serialization failed: %d card(s) referenced by the requested collections "
- "are saved outside them, which would produce an incomplete export. See the "
- "warnings above for the affected entities. Pass continue-on-error to export "
- "anyway, skipping the affected dashboards and cards.")
- (count (distinct (map (comp :id :escapee) escaped))))
- {:status-code 400}))))
+(defn- analytics-card-ids
+ "Of `card-ids`, the subset living in an 'analytics' namespace collection. These are removed from extraction (they
+ have stable entity_ids across instances) and references to them stay valid, resolving on import. `card-ids` is
+ queried in bounded `:in` batches so a large closure doesn't blow past database parameter limits."
+ [card-ids]
+ (let [analytics-colls (analytics-collection-ids)]
+ (if (and (seq card-ids) (seq analytics-colls))
+ (into #{}
+ (comp (partition-all serdes/query-batch-size)
+ (mapcat (fn [batch]
+ (t2/select-pks-set :model/Card {:where [:and
+ [:in :id (vec batch)]
+ [:in :collection_id (vec analytics-colls)]]}))))
+ card-ids)
+ #{})))
(defn- resolve-targets
"Returns all targets (for either supplied initial `targets` or for supplied `user-id`)."
@@ -200,16 +140,15 @@
(resolve-targets opts user-id))
;; `by-model` is a map of `{model-name [ids ...]}`
by-model (u/group-by first second (keys nodes))
- ;; Escape analysis only matters when exporting collection content — it checks whether
- ;; cards referenced by dashboards live outside the target collections.
- {:keys [reportable-escaped analytics-card-ids]} (when has-content?
- (escape-analysis by-model nodes))]
- ;; A card referenced from inside the requested collections but living outside them would produce an incomplete
- ;; export.
- (when (seq reportable-escaped)
- (handle-escapees! reportable-escaped (:continue-on-error opts))) ;; may throw
- (let [coll-set (get by-model "Collection")
- ;; When targets are specified, also include Tables found via descendants
+ ;; Cards in analytics collections aren't extracted (stable entity_ids), but references to them stay valid.
+ analytics-cards (when has-content?
+ (analytics-card-ids (get by-model "Card")))
+ coll-set (get by-model "Collection")]
+ ;; Validate that every entity to be extracted has all its references satisfied; aborts (unless
+ ;; continue-on-error) before any archive is produced, so incomplete exports fail fast and loudly.
+ (when has-content?
+ (dependency-validation/validate-dependencies! by-model coll-set analytics-cards opts)) ;; may throw
+ (let [;; When targets are specified, also include Tables found via descendants
;; (published tables in target collections). These are extracted by ID, not all.
targeted-data-model (when (seq targets)
(select-keys by-model serdes.models/data-model-in-collection))
@@ -218,8 +157,8 @@
(seq targeted-data-model) (merge targeted-data-model)
;; Remove analytics cards from extraction - they have stable entity_ids across instances
;; so cards that reference them can still be exported and imported correctly
- (and analytics-card-ids (contains? by-model "Card"))
- (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))
+ (and analytics-cards (contains? by-model "Card"))
+ (update "Card" (fn [ids] (vec (remove analytics-cards ids)))))
extract-by-ids (fn [[model ids]]
(serdes/extract-all model (merge opts {:collection-set coll-set
:where [:in :id ids]})))
@@ -243,6 +182,8 @@
(merge
(u/traverse colls #(serdes/ascendants (first %) (second %)))
(u/traverse colls #(serdes/descendants (first %) (second %) {})))))
- (def escaped (escape-analysis (u/group-by first second (keys nodes)) nodes))
- ;; continue-on-error? true so this just logs in the REPL instead of throwing
- (handle-escapees! (:reportable-escaped escaped) true))
+ (def by-model (u/group-by first second (keys nodes)))
+ ;; continue-on-error so this just logs in the REPL instead of throwing
+ (dependency-validation/validate-dependencies! by-model (get by-model "Collection")
+ (analytics-card-ids (get by-model "Card"))
+ {:continue-on-error true}))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index ca6642beb17..d9cad571a2e 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -172,7 +172,7 @@
;; `::strip` is handled in `metabase.serialization`
(circular path) (assoc ::serdes/strip (keys-to-strip ingested)))
;; we need less deps when trying to load "stripped" data
- deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))
+ deps (->> (serdes/deserialization-dependencies (apply dissoc ingested (::serdes/strip ingested)))
(remove seen))
_ (when (seq deps)
(log/debug "Loading dependencies"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index b7d5e225c67..9a2db8279bf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -239,11 +239,11 @@
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))
(deftest export-log-captures-extract-warnings-test
- (testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)"
- ;; A dashboard in the exported collection references a card living in a different collection. Escape analysis
- ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. Under
- ;; continue-on-error the export still completes, and that warning must land in export.log even though extract
- ;; happens outside the storage logging block.
+ (testing "export.log captures dependency-validation warnings emitted during the eager extract phase (GHY-3802)"
+ ;; A dashboard in the exported collection references a card living in a different collection. Dependency
+ ;; validation runs eagerly inside extract/extract (before storage streaming) and warns about the unsatisfied
+ ;; reference. Under continue-on-error the export still completes, and that warning must land in export.log even
+ ;; though extract happens outside the storage logging block.
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Collection target {:name "Target Collection"}
:model/Collection other {:name "Other Collection"}
@@ -256,8 +256,8 @@
:continue_on_error true))
log (read-export-log res)]
(is (some? log) "export.log should be present in the archive")
- (is (re-find #"outside requested collections" log)
- "export.log should contain the escape-analysis warning emitted during extract"))))))
+ (is (re-find #"not included in the export" log)
+ "export.log should contain the dependency-validation warning emitted during extract"))))))
(deftest export-aborts-on-escaped-card-test
(testing "Export fails loudly with a 4xx instead of silently emitting an empty archive when a referenced card lives outside the requested collections (#75176)"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.clj
new file mode 100644
index 00000000000..d9c53a56454
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.clj
@@ -0,0 +1,254 @@
+(ns metabase-enterprise.serialization.v2.dependency-validation-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.test-util :as ts]
+ [metabase-enterprise.serialization.v2.extract :as extract]
+ [metabase.documents.prose-mirror :as prose-mirror]
+ [metabase.models.serialization :as serdes]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :each (fn [f] (search.tu/with-index-disabled (f))))
+
+(defn- by-model [model-name extraction]
+ (->> extraction
+ (into [])
+ (filter #(= model-name ((comp :model last :serdes/meta) %)))))
+
+(defn- ids-by-model [model-name extraction]
+ (->> (by-model model-name extraction)
+ (map (comp :id last :serdes/meta))
+ set))
+
+(defn- extract-aborts!
+ "Realize `(extract/extract opts)`, asserting it aborts because a reference can't be satisfied (#75176, GHY-4010).
+ Warnings are logged before the abort, so callers can still assert on the logged messages."
+ [opts]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"incomplete export"
+ (into [] (extract/extract opts)))))
+
+(deftest escape-report-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}
+ :model/Collection {coll2-id :id} {:name "Other Collection"}
+ :model/Collection {coll3-id :id} {:name "Third Collection"}
+ :model/Collection {clean-coll-id :id
+ clean-coll-eid :entity_id} {:name "Clean Collection"}
+ :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}
+ ;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
+ :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}
+ :model/Card {clean-card-eid :entity_id} {:name "Clean Card"
+ :collection_id clean-coll-id
+ :database_id db-id}
+ :model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}
+ :model/Card _ {:name "Dependent Card"
+ :collection_id coll2-id
+ :dataset_query {:type :query
+ :database db-id
+ :query {:source-table (str "card__" card1-id)}}}
+ :model/User user {:email "[email protected]"}
+ :model/Collection pcoll {:name "Personal Collection"
+ :personal_owner_id (:id user)}
+ :model/Card pcard {:name "Personal Card"
+ :collection_id (:id pcoll)}
+ :model/Card _ {:name "External Card"
+ :dataset_query {:database db-id
+ :type :query
+ :query {:source-table (str "card__" (:id pcard))}}}
+ :model/Card _ {:name "Card with parameters"
+ :collection_id coll3-id
+ :parameters [{:id "abc"
+ :type "category"
+ :values_source_type "card"
+ :values_source_config {:card_id card1-id}}]}]
+ (testing "Complain about card not available for exporting"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (extract-aborts! {:targets [["Collection" coll1-id]]
+ :no-settings true
+ :no-data-model true})
+ (is (some #(re-find #"not included in the export" %)
+ (into #{}
+ (map :message)
+ (messages))))))
+ (testing "Complain about card depending on an outside card: "
+ (testing "when its :source-table"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (extract-aborts! {:targets [["Collection" coll2-id]]
+ :no-settings true
+ :no-data-model true})
+ (is (some #(re-find #"not included in the export" %)
+ (into #{}
+ (map :message)
+ (messages))))))
+ (testing "when it's :parameters"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ ;; the parameter-referencing "Card with parameters" lives in coll3, so target coll3 to exercise it
+ (extract-aborts! {:targets [["Collection" coll3-id]]
+ :no-settings true
+ :no-data-model true})
+ (is (some #(re-find #"not included in the export" %)
+ (into #{}
+ (map :message)
+ (messages)))))))
+ (testing "When exporting all collections"
+ (testing "Complain about dependents in personal collections"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (extract-aborts! {:no-settings true
+ :no-data-model true})
+ (is (some #(re-find #"not included in the export" %)
+ (into #{}
+ (map :message)
+ (messages)))))))
+ (testing "exporting a clean collection works even when other collections have escape issues"
+ (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]
+ :no-settings true
+ :no-data-model true
+ :no-transforms true}))]
+ (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))
+ (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))
+ (testing "data-model-only export is not blocked by dependency validation"
+ (let [extracted (into [] (extract/extract {:no-collections true
+ :no-settings true
+ :no-transforms true}))]
+ (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))
+ "Databases should be exported even when cards reference personal collections"))))))
+
+(deftest escape-continue-on-error-test
+ (testing "continue-on-error lets the export proceed past dependency validation instead of aborting (#74622)"
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
+ :model/Collection {coll-id :id
+ coll-eid :entity_id} {:name "Target Collection"}
+ :model/Card {clean-eid :entity_id} {:name "Clean Card"
+ :collection_id coll-id
+ :database_id db-id}
+ :model/Dashboard {dash-id :id
+ dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}
+ ;; this card lives outside the target collection, so it "escapes"
+ :model/Card {escaped-eid :entity_id
+ escaped-id :id} {:name "Escaped Card" :database_id db-id}
+ :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]
+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]
+ (testing "without the flag, dependency validation aborts the whole export with an error (#75176)"
+ (extract-aborts! opts))
+ (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]
+ (is (= #{coll-eid} (ids-by-model "Collection" extracted)))
+ (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))
+ (is (contains? (ids-by-model "Card" extracted) clean-eid)
+ "the clean card inside the target collection is exported")
+ (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))
+ "the escaped card outside the target collection is not exported")
+ (is (some #(re-find #"not included in the export" %)
+ (map :message (messages)))
+ "the unsatisfied reference is still logged as a warning")))))))))
+
+(deftest dependency-validation-missing-data-ref-test
+ (testing "Export aborts when a card references a data-model row that no longer exists in the appdb (GHY-4010)"
+ ;; The card still serializes a portable reference by looking the row up; with the row gone that reference would be
+ ;; malformed, so the export must fail fast. Covered across the reference kinds the old lib-walker-only derivation
+ ;; missed: a Field or Segment in the query, and a Field reachable only through visualization_settings (a
+ ;; JSON-encoded ref the query walkers don't see). (continue-on-error's abort suppression is covered by
+ ;; escape-continue-on-error-test.)
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
+ :model/Table {table-id :id} {:db_id db-id :name "T"}
+ :model/Collection {coll-id :id} {:name "Target Collection"}]
+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true :no-transforms true}]
+ (doseq [{:keys [label query-filter viz-settings delete! msg-re]}
+ [{:label "a Field referenced in the query"
+ :query-filter (fn [field-id _seg-id] [:> [:field field-id nil] 1])
+ :delete! (fn [field-id _seg-id] (t2/delete! :model/Field field-id))
+ :msg-re #"Field .* is missing from the source database"}
+ {:label "a Segment referenced in the query"
+ :query-filter (fn [_field-id seg-id] [:segment seg-id])
+ :delete! (fn [_field-id seg-id] (t2/delete! :model/Segment seg-id))
+ :msg-re #"Segment .* is missing from the source database"}
+ {:label "a Field referenced only in visualization_settings"
+ :viz-settings (fn [field-id] {:column_settings {(str "[\"ref\",[\"field\"," field-id ",null]]")
+ {:column_title "X"}}})
+ :delete! (fn [field-id _seg-id] (t2/delete! :model/Field field-id))
+ :msg-re #"Field .* is missing from the source database"}]]
+ (testing label
+ (ts/with-temp-dpc [:model/Field {field-id :id} {:table_id table-id :name "F" :base_type :type/Integer}
+ :model/Segment {segment-id :id} {:table_id table-id :name "Seg" :definition {}}
+ :model/Card _ {:name "Card"
+ :collection_id coll-id
+ :database_id db-id
+ :table_id table-id
+ :query_type :query
+ :dataset_query (cond-> {:database db-id
+ :type :query
+ :query {:source-table table-id}}
+ query-filter
+ (assoc-in [:query :filter]
+ (query-filter field-id segment-id)))
+ :visualization_settings (if viz-settings
+ (viz-settings field-id)
+ {})}]
+ (testing "with the reference present, the export succeeds"
+ (is (seq (into [] (extract/extract opts)))))
+ (delete! field-id segment-id)
+ (testing "after the reference is deleted, the export aborts and names it"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (extract-aborts! opts)
+ (is (some #(re-find msg-re %) (map :message (messages)))
+ "the warning reports the unsatisfied data-model reference")))))))))))
+
+(deftest serialization-dependencies-content-models-test
+ (testing "serialization-dependencies derives export deps from the raw entity for the newly-covered content models (GHY-4010)"
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
+ :model/Table {table-id :id} {:db_id db-id :name "T"}
+ :model/Field {field-id :id} {:table_id table-id :name "F" :base_type :type/Integer}
+ :model/Collection {parent-id :id} {:name "Parent"}
+ :model/Collection {child-id :id} {:name "Child" :location (str "/" parent-id "/")}
+ :model/Collection {snip-coll-id :id} {:name "Snips" :namespace "snippets"}
+ :model/Card {model-card-id :id} {:name "Model" :type :model :database_id db-id}
+ :model/Card {embed-card-id :id} {:name "Embedded" :database_id db-id}]
+ (let [deps (fn [model id]
+ (serdes/serialization-dependencies model (t2/select-one (keyword "model" model) :id id)))]
+ (testing "Collection derives its parent from the raw :location path"
+ (is (= #{[{:model "Collection" :id parent-id}]} (deps "Collection" child-id)))
+ (is (empty? (deps "Collection" parent-id)) "a root collection has no parent dependency"))
+ (testing "Timeline references its containing collection"
+ (let [tl-id (t2/insert-returning-pk! :model/Timeline {:name "TL" :icon "star" :collection_id child-id :creator_id (mt/user->id :rasta)})]
+ (is (= #{[{:model "Collection" :id child-id}]} (deps "Timeline" tl-id)))))
+ (testing "NativeQuerySnippet references its containing collection"
+ (let [snip-id (t2/insert-returning-pk! :model/NativeQuerySnippet
+ {:name "snip" :content "1=1" :collection_id snip-coll-id
+ :creator_id (mt/user->id :rasta)})]
+ (is (= #{[{:model "Collection" :id snip-coll-id}]} (deps "NativeQuerySnippet" snip-id)))))
+ (testing "Action (query) references its model Card, Database, and the tables/fields in its query"
+ (let [action-id (t2/insert-returning-pk! :model/Action {:name "A" :type :query :model_id model-card-id})]
+ (t2/insert! :model/QueryAction {:action_id action-id
+ :database_id db-id
+ :dataset_query {:database db-id
+ :type :query
+ :query {:source-table table-id
+ :filter [:> [:field field-id nil] 1]}}})
+ (is (= #{[{:model "Card" :id model-card-id}]
+ [{:model "Database" :id db-id}]
+ [{:model "Table" :id table-id}]
+ [{:model "Field" :id field-id}]}
+ (deps "Action" action-id)))))
+ (testing "Document references its embedded cards and its containing collection"
+ (let [doc-id (t2/insert-returning-pk! :model/Document
+ {:name "D"
+ :collection_id child-id
+ :creator_id (mt/user->id :rasta)
+ :content_type prose-mirror/prose-mirror-content-type
+ :document {:type "doc"
+ :content [{:type "cardEmbed"
+ :attrs {:id embed-card-id}}]}})]
+ (is (= #{[{:model "Collection" :id child-id}]
+ [{:model "Card" :id embed-card-id}]}
+ (deps "Document" doc-id))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 71d5ca04052..7099fcbda06 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -526,7 +526,7 @@
[{:id model-eid :model "Card"}]
[{:id card-eid :model "Card"}]
[{:id "Linked database" :model "Database"}]}
- (set (serdes/dependencies extracted-dashboard))))
+ (set (serdes/deserialization-dependencies extracted-dashboard))))
(storage/store! (seq extraction) (storage.files/file-writer dump-dir))))
(testing "ingest and load"
;; ingest
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 3f5f6c8daab..c507a46b1e0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -330,7 +330,7 @@
(testing "cards depend on their Collection and the query's Database (not the Tables/Fields it references)"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser))))))
+ (set (serdes/deserialization-dependencies ser))))))
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c2-id))]
(is (=? {:serdes/meta [{:model "Card" :id c2-eid :label "second_question"}]
:creator_id "[email protected]"
@@ -349,7 +349,7 @@
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]
[{:model "Card" :id c1-eid}]}
- (set (serdes/dependencies ser))))))
+ (set (serdes/deserialization-dependencies ser))))))
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c3-id))]
(is (=? {:serdes/meta [{:model "Card" :id c3-eid :label "third_question"}]
:creator_id "[email protected]"
@@ -382,7 +382,7 @@
(testing "cards depend on their Database (kept, query empty) and Collection (not the Fields in visualization_settings)"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "Cards can be based on other cards"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c5-id))]
(is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]
@@ -400,7 +400,7 @@
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]
[{:model "Card" :id c4-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "Dashboards include their Dashcards"
(let [ser (ts/extract-one "Dashboard" other-dash-id)]
(is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]
@@ -433,7 +433,7 @@
[{:model "Action" :id action-eid}]
[{:model "Database" :id "My Database"}]
[{:model "Collection" :id dave-coll-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "Dashboards with parameters where the source is a card"
(let [ser (ts/extract-one "Dashboard" param-dash-id)]
(is (=? {:parameters
@@ -450,7 +450,7 @@
[{:model "Card" :id c1-eid}]
;; the parameter's value_field references a Field, but only its Database is a dependency
[{:model "Database", :id "My Database"}]}
- (set (serdes/dependencies ser))))))
+ (set (serdes/deserialization-dependencies ser))))))
(testing "Cards with parameters where the source is a card"
(let [ser (ts/extract-one "Dashboard" param-dash-id)]
(is (=? {:parameters
@@ -467,7 +467,7 @@
[{:model "Card" :id c1-eid}]
;; the parameter's value_field references a Field, but only its Database is a dependency
[{:model "Database", :id "My Database"}]}
- (set (serdes/dependencies ser))))))
+ (set (serdes/deserialization-dependencies ser))))))
(testing "collection filtering based on :user option"
(testing "only unowned collections are returned with no user"
(is (= ["Some Collection"]
@@ -526,7 +526,7 @@
[{:model "Card" :id c2-eid}]
[{:model "Card" :id c3-eid}]
[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))
(deftest dimensions-test
(mt/with-empty-h2-app-db!
@@ -575,7 +575,7 @@
(->> ser :dimensions (map :entity_id)))))
(testing "depend only on the Database; the Table is synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "foreign key dimensions are inlined into their Fields"
(let [ser (ts/extract-one "Field" fk-id)]
(is (malli= [:map
@@ -597,7 +597,7 @@
(:dimensions ser))))
(testing "depend only on the Database; the Table, FK target and human-readable Fields are synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))
(deftest native-query-snippets-test
(mt/with-empty-h2-app-db!
@@ -644,7 +644,7 @@
(is (not (contains? ser :id)))
(testing "and depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser)))))
+ (set (serdes/deserialization-dependencies ser)))))
(testing "and will bring collection to extraction"
(is (= {["Collection" coll-id] {"NativeQuerySnippet" s1-id}}
(serdes/required "NativeQuerySnippet" s1-id))))))
@@ -660,7 +660,7 @@
ser))
(is (not (contains? ser :id)))
(testing "and has no deps"
- (is (empty? (serdes/dependencies ser))))))
+ (is (empty? (serdes/deserialization-dependencies ser))))))
(testing "Snippet collection is exported when snippet is exported as a card dep (#51901)"
(is (= {["Collection" coll2-id] nil
["Card" card-id] {"Collection" coll2-id}
@@ -764,7 +764,7 @@
(is (not (contains? ser :id)))
(testing "depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "with events"
(let [ser (ts/extract-one "Timeline" line-id)]
(is (=? {:serdes/meta [{:model "Timeline" :id line-eid :label "populated_timeline"}]
@@ -779,7 +779,7 @@
(is (not (contains? (-> ser :events first) :id)))
(testing "depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser)))))))))))
+ (set (serdes/deserialization-dependencies ser)))))))))))
(deftest segments-test
(mt/with-empty-h2-app-db!
@@ -812,7 +812,7 @@
(is (not (contains? ser :id)))
(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))
(defn- mbql5-measure-definition
"Create an MBQL5 measure definition with a sum aggregation."
@@ -853,7 +853,7 @@
(is (not (contains? ser :id)))
(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser))))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))))
(deftest measure-referencing-measure-test
(mt/with-empty-h2-app-db!
@@ -894,7 +894,7 @@
:aggregation [[:* {} [:measure {} m1-eid] 2]]}]}}
ser))
(testing "depends on the referenced Measure"
- (is (contains? (set (serdes/dependencies ser))
+ (is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Measure" :id m1-eid}]))))))))))))
(deftest measure-referencing-segment-test
@@ -936,7 +936,7 @@
:aggregation [[:count-where {} [:segment {} seg-eid]]]}]}}
ser))
(testing "depends on the referenced Segment"
- (is (contains? (set (serdes/dependencies ser))
+ (is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Segment" :id seg-eid}]))))))))))
(deftest table-publishing-serdes-test
@@ -959,7 +959,7 @@
(testing "collection_id is transformed to entity_id"
(is (= coll-eid (:collection_id ser))))
(testing "depends on the collection"
- (is (contains? (set (serdes/dependencies ser))
+ (is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Collection" :id coll-eid}])))))
(testing "unpublished table without collection_id"
(let [ser (ts/extract-one "Table" unpub-table-id)]
@@ -969,7 +969,7 @@
(is (nil? (:collection_id ser))))
(testing "does not depend on any collection"
(is (not (some #(= "Collection" (:model (first %)))
- (serdes/dependencies ser)))))))
+ (serdes/deserialization-dependencies ser)))))))
(testing "regular table without publishing fields set"
(let [ser (ts/extract-one "Table" table-id)]
(testing "is_published is omitted (default false)"
@@ -1011,7 +1011,7 @@
(is (not (contains? ser :id)))
(testing "depends on the Model"
(is (= #{[{:model "Card" :id card-eid-1}]}
- (set (serdes/dependencies ser)))))))))))))
+ (set (serdes/deserialization-dependencies ser)))))))))))))
(deftest query-action-test
(mt/with-empty-h2-app-db!
@@ -1054,7 +1054,7 @@
(testing "depends on the Model and Database"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Card" :id card-eid-1}]}
- (set (serdes/dependencies ser)))))))))))))
+ (set (serdes/deserialization-dependencies ser)))))))))))))
(deftest field-values-test
(mt/with-empty-h2-app-db!
@@ -1093,7 +1093,7 @@
":field_id is dropped; its implied by the path")
(testing "depend only on the Database; the parent Field is synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "extract-metabase behavior"
(testing "without :include-field-values"
(is (= #{}
@@ -1125,7 +1125,7 @@
":field_id is dropped; its implied by the path")
(testing "depend only on the Database; the parent Field is synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "extract-metabase behavior"
(let [models (->> {} (extract/extract) (map (comp :model last :serdes/meta)))]
(is (= 1
@@ -1183,7 +1183,7 @@
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid-2}]
[{:model "Card" :id card-eid-1}]}
- (set (serdes/dependencies ser))))))
+ (set (serdes/deserialization-dependencies ser))))))
(testing "Nullable transformations are omitted"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id card-id-2))]
(is (not (contains? ser :made_public_by_id))))))))
@@ -1465,8 +1465,8 @@
(let [msgs (into #{}
(map :message)
(messages))]
- (is (some #(str/starts-with? % "Failed to export Dashboard") msgs))
- (is (some #(str/starts-with? % "Failed to export Cards") msgs))))))))))
+ (is (some #(re-find #"not included in the export" %) msgs)
+ "the export warns that a referenced personal-collection card is not included")))))))))
(deftest click-behavior-references-to-deleted-cards
(mt/with-empty-h2-app-db!
@@ -1570,124 +1570,6 @@
{:model "Field", :id "Nested Field"}]
(:serdes/meta (ts/extract-one "Field" nested-id))))))))
-(deftest escape-report-test
- (mt/with-empty-h2-app-db!
- (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}
- :model/Collection {coll2-id :id} {:name "Other Collection"}
- :model/Collection {coll3-id :id} {:name "Third Collection"}
- :model/Collection {clean-coll-id :id
- clean-coll-eid :entity_id} {:name "Clean Collection"}
- :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}
- ;; non-H2 engine so the database survives serdes extract filtering
- :model/Database {db-id :id} {:engine :postgres}
- :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}
- :model/Card {clean-card-eid :entity_id} {:name "Clean Card"
- :collection_id clean-coll-id
- :database_id db-id}
- :model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}
- :model/Card _ {:name "Dependent Card"
- :collection_id coll2-id
- :dataset_query {:type :query
- :database db-id
- :query {:source-table (str "card__" card1-id)}}}
- :model/User user {:email "[email protected]"}
- :model/Collection pcoll {:name "Personal Collection"
- :personal_owner_id (:id user)}
- :model/Card pcard {:name "Personal Card"
- :collection_id (:id pcoll)}
- :model/Card _ {:name "External Card"
- :dataset_query {:database db-id
- :type :query
- :query {:source-table (str "card__" (:id pcard))}}}
- :model/Card _ {:name "Card with parameters"
- :collection_id coll3-id
- :parameters [{:id "abc"
- :type "category"
- :values_source_type "card"
- :values_source_config {:card_id card1-id}}]}]
- (testing "Complain about card not available for exporting"
- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract-aborts! {:targets [["Collection" coll1-id]]
- :no-settings true
- :no-data-model true})
- (is (some #(str/starts-with? % "Failed to export Dashboard")
- (into #{}
- (map :message)
- (messages))))))
- (testing "Complain about card depending on an outside card: "
- (testing "when its :source-table"
- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract-aborts! {:targets [["Collection" coll2-id]]
- :no-settings true
- :no-data-model true})
- (is (some #(str/starts-with? % "Failed to export Cards")
- (into #{}
- (map :message)
- (messages))))))
- (testing "when it's :parameters"
- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract-aborts! {:targets [["Collection" coll2-id]]
- :no-settings true
- :no-data-model true})
- (is (some #(str/starts-with? % "Failed to export Cards")
- (into #{}
- (map :message)
- (messages)))))))
- (testing "When exporting all collections"
- (testing "Complain about dependents in personal collections"
- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract-aborts! {:no-settings true
- :no-data-model true})
- (is (some #(str/starts-with? % "Failed to export Cards")
- (into #{}
- (map :message)
- (messages)))))))
- (testing "exporting a clean collection works even when other collections have escape issues"
- (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]
- :no-settings true
- :no-data-model true
- :no-transforms true}))]
- (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))
- (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))
- (testing "data-model-only export is not blocked by escape analysis"
- (let [extracted (into [] (extract/extract {:no-collections true
- :no-settings true
- :no-transforms true}))]
- (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))
- "Databases should be exported even when cards reference personal collections"))))))
-
-(deftest escape-continue-on-error-test
- (testing "continue-on-error lets the export proceed past escape analysis instead of aborting (#74622)"
- (mt/with-empty-h2-app-db!
- (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering
- :model/Database {db-id :id} {:engine :postgres}
- :model/Collection {coll-id :id
- coll-eid :entity_id} {:name "Target Collection"}
- :model/Card {clean-eid :entity_id} {:name "Clean Card"
- :collection_id coll-id
- :database_id db-id}
- :model/Dashboard {dash-id :id
- dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}
- ;; this card lives outside the target collection, so it "escapes"
- :model/Card {escaped-eid :entity_id
- escaped-id :id} {:name "Escaped Card" :database_id db-id}
- :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]
- (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]
- (testing "without the flag, escape analysis aborts the whole export with an error (#75176)"
- (extract-aborts! opts))
- (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"
- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]
- (is (= #{coll-eid} (ids-by-model "Collection" extracted)))
- (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))
- (is (contains? (ids-by-model "Card" extracted) clean-eid)
- "the clean card inside the target collection is exported")
- (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))
- "the escaped card outside the target collection is not exported")
- (is (some #(str/starts-with? % "Failed to export Dashboard")
- (map :message (messages)))
- "the escape report is still logged as a warning")))))))))
-
(deftest recursive-colls-test
(mt/with-empty-h2-app-db!
(mt/with-temp [:model/Collection {parent-id :id
@@ -1951,7 +1833,7 @@
(testing "metabot depends on its model entities and collection"
(is (= #{[{:model "Card" :id model-eid}]
[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser)))))
+ (set (serdes/deserialization-dependencies ser)))))
(testing "metabot storage-path uses top-level metabots directory"
(is (= [{:label "metabots"} {:label "Test Metabot" :key metabot-eid}]
(serdes/storage-path ser {})))))))))
@@ -1996,7 +1878,7 @@
(testing "metabot depends on its prompts' cards and its collection"
(is (= #{[{:model "Card" :id card-eid}]
[{:model "Collection" :id coll-eid}]}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))
(deftest document-test
(mt/with-empty-h2-app-db!
@@ -2058,7 +1940,7 @@
[{:model "Dashboard" :id (:entity_id dashboard)}]
[{:model "Card" :id (:entity_id linked-card)}]
(serdes/generate-path "Table" table)}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/deserialization-dependencies ser))))))))))
(deftest visualizer-dashboard-card-settings-test
(testing "visualizer settings transform entity IDs <-> card IDs"
@@ -2136,7 +2018,7 @@
:created_at string?}
ser))
(is (not (contains? ser :id)))
- (is (empty? (serdes/dependencies ser)))))
+ (is (empty? (serdes/deserialization-dependencies ser)))))
(testing "custom tags extract correctly"
(let [ser (serdes/extract-one "TransformTag" {} (t2/hydrate (t2/select-one :model/TransformTag :id custom-tag-id) :tags))]
(is (=? {:serdes/meta [{:model "TransformTag"
@@ -2146,7 +2028,7 @@
ser))
(is (not (contains? ser :built_in_type)))
(is (not (contains? ser :id)))
- (is (empty? (serdes/dependencies ser)))))
+ (is (empty? (serdes/deserialization-dependencies ser)))))
(testing "all transform tags are extracted"
(is (= #{hourly-tag-eid daily-tag-eid custom-tag-eid}
(ids-by-model "TransformTag" (extract/extract {}))))))))))
@@ -2255,7 +2137,7 @@
(is (= [hourly-tag-eid custom-tag-eid daily-tag-eid] tag-ids))
(is (= [0 1 2] positions))))
(testing "dependencies include collection, source database, and tags (the source Table itself is not a dependency)"
- (let [deps (set (serdes/dependencies ser))]
+ (let [deps (set (serdes/deserialization-dependencies ser))]
(is (contains? deps [{:model "Collection" :id coll-eid}]))
(is (contains? deps [{:model "Database" :id "My Database"}]))
(is (not (contains? deps [{:model "Database" :id "My Database"}
@@ -2317,7 +2199,7 @@
(is (= "SELECT 1" (get-in ser [:source :query :stages 0 :native]))))
(testing "Transform/dependencies does not emit a Database dep when source_database_id is nil"
(is (not-any? #(some (fn [{:keys [model]}] (= "Database" model)) %)
- (serdes/dependencies (assoc reloaded :tags [])))))))))))
+ (serdes/deserialization-dependencies (assoc reloaded :tags [])))))))))))
(deftest table-with-transform-id-dependency-test
(testing "Table created by a Transform declares the Transform as a serdes dependency (GDGT-2444)"
@@ -2349,7 +2231,7 @@
(testing "transform_id is transformed to entity_id"
(is (= transform-eid (:transform_id ser))))
(testing "depends on the transform"
- (is (contains? (set (serdes/dependencies ser))
+ (is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Transform" :id transform-eid}])))))))))
(deftest transform-job-extraction-test
@@ -2420,7 +2302,7 @@
(is (= 0 (-> ser :job_tags first :position))))
(testing "dependencies include referenced tags"
(is (= #{[{:model "TransformTag" :id hourly-tag-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "custom job extracts correctly"
(let [ser (serdes/extract-one "TransformJob" {} (t2/hydrate (t2/select-one :model/TransformJob :id custom-job-id) :job_tags))]
(is (=? {:serdes/meta [{:model "TransformJob"
@@ -2441,7 +2323,7 @@
(testing "dependencies include all referenced tags"
(is (= #{[{:model "TransformTag" :id custom-tag-eid}]
[{:model "TransformTag" :id daily-tag-eid}]}
- (set (serdes/dependencies ser)))))))
+ (set (serdes/deserialization-dependencies ser)))))))
(testing "all transform jobs are extracted"
(is (= #{hourly-job-eid custom-job-eid}
(ids-by-model "TransformJob" (extract/extract {}))))))))))
@@ -2543,7 +2425,7 @@
ser))
(is (not (contains? ser :id)))
(testing "has no dependencies"
- (is (empty? (serdes/dependencies ser)))))))))
+ (is (empty? (serdes/deserialization-dependencies ser)))))))))
(deftest custom-viz-plugin-test
(mt/with-empty-h2-app-db!
@@ -2570,7 +2452,7 @@
(is (not (contains? ser :id)))
(is (not (contains? ser :status)))
(testing "has no dependencies"
- (is (empty? (serdes/dependencies ser)))))))))
+ (is (empty? (serdes/deserialization-dependencies ser)))))))))
(deftest ^:parallel export-parameters-sorts-by-id-test
(let [params [{:id "zebra" :name "Z param" :type :category}
@@ -2770,7 +2652,7 @@
:dataset_query (mbql5-query db-id table-id)
:display :table}]
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card card-id))
- deps (serdes/dependencies ser)]
+ deps (serdes/deserialization-dependencies ser)]
;; Database dep comes from mbql-deps on the query's :database key
(is (contains? (set deps) [{:model "Database" :id "Test DB"}])
"Database dependency should come from the query")
@@ -2841,7 +2723,7 @@
ser))
(is (not (contains? ser :id)))
(testing "embedding themes have no dependencies"
- (is (empty? (serdes/dependencies ser))))))
+ (is (empty? (serdes/deserialization-dependencies ser))))))
(testing "all embedding themes are extracted"
(is (= #{light-eid dark-eid}
(ids-by-model "EmbeddingTheme" (extract/extract {}))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index aa0f7a93c9d..d2d5b122048 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -609,7 +609,7 @@
derived-measure))
(is (= #{[{:id "my-db", :model "Database"}]
[{:id (:entity_id @msr1s), :model "Measure"}]}
- (serdes/mbql-deps (:definition derived-measure))))))
+ (serdes/mbql-deps false (:definition derived-measure))))))
(testing "deserializing adjusts the measure IDs properly"
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
index a5000959b2f..c79140844ad 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj
@@ -138,7 +138,7 @@
(def ^:private auto-synthesized-fk-targets
"FK target models whose importers auto-create a stub entity if the target is missing, so
- they don't have to be declared in `serdes/dependencies`. Everything else MUST be declared
+ they don't have to be declared in `serdes/deserialization-dependencies`. Everything else MUST be declared
— otherwise `*import-fk*` throws \"Could not find foreign key target\" and aborts the
whole import (see GDGT-2444 for an example)."
#{:model/User :model/Table :model/Field})
@@ -174,7 +174,7 @@
(deftest ^:parallel serialization-direct-fk-dependencies-completeness-test
(testing (str "Every direct FK declared via `(serdes/fk ...)` in a loadable (non-inlined) "
- "model's `make-spec` is surfaced in that model's `serdes/dependencies` — "
+ "model's `make-spec` is surfaced in that model's `serdes/deserialization-dependencies` — "
"otherwise `*import-fk*` will throw \"Could not find foreign key target\" at "
"import time (see GDGT-2444).")
(let [inlined? (set serdes.models/inlined-models)]
@@ -194,13 +194,13 @@
(let [entity (into {:serdes/meta [{:model m :id "self"}]}
(for [[field _] fks]
[field (str "fake-eid-" (name field))]))
- deps (set (serdes/dependencies entity))
+ deps (set (serdes/deserialization-dependencies entity))
dep-models (set (keep (comp :model peek) deps))]
(doseq [[field target] fks]
- (testing (format "FK `%s` -> `%s` must appear in `(serdes/dependencies)`"
+ (testing (format "FK `%s` -> `%s` must appear in `(serdes/deserialization-dependencies)`"
field target)
(is (contains? dep-models (name target))
- (format (str "`(serdes/dependencies %s)` did not include a `%s` entry "
+ (format (str "`(serdes/deserialization-dependencies %s)` did not include a `%s` entry "
"when `%s` was populated. Add the FK to the model's "
"`dependencies` method (see Table/Transform for an example).")
m (name target) field))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj
index 1a47657caab..c5a49ce43b0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj
@@ -60,7 +60,7 @@
extracted))
(is (not (contains? extracted :entity_id))))
(testing "depends on the referenced Card"
- (is (= [[{:model "Card" :id card-eid}]] (serdes/dependencies extracted))))
+ (is (= [[{:model "Card" :id card-eid}]] (serdes/deserialization-dependencies extracted))))
(testing "importing resolves the path back to the local card row"
(t2/delete! :model/OsiAiContext :entity_type "card" :entity_local_id card-id)
(serdes.load/load-metabase! (ingestion-in-memory [extracted]))
@@ -86,7 +86,7 @@
(is (=? {:serdes/meta [{:model serdes-model :id eid} {:model "OsiAiContext" :id "ai_context"}]}
extracted)))
(testing "depends on the referenced entity"
- (is (= [[{:model serdes-model :id eid}]] (serdes/dependencies extracted))))
+ (is (= [[{:model serdes-model :id eid}]] (serdes/deserialization-dependencies extracted))))
(testing "importing resolves the path back to the local id"
(t2/delete! :model/OsiAiContext :entity_type entity-type :entity_local_id id)
(serdes.load/load-metabase! (ingestion-in-memory [extracted]))
@@ -102,7 +102,7 @@
(is (= {:model "OsiAiContext" :id "ai_context"} (self-segment extracted)))
(is (= "Table" (:model (last (pop (vec (serdes/path extracted))))))))
(testing "depends on the Table"
- (is (= [(vec (pop (vec (serdes/path extracted))))] (serdes/dependencies extracted))))
+ (is (= [(vec (pop (vec (serdes/path extracted))))] (serdes/deserialization-dependencies extracted))))
(testing "importing resolves the path back to the local id"
(t2/delete! :model/OsiAiContext :entity_type "table" :entity_local_id table-id)
(serdes.load/load-metabase! (ingestion-in-memory [extracted]))
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 0786e5bfc89..bfc1f0a87b6 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -457,7 +457,7 @@
:import serdes/import-visualization-settings}}
:defaults {:archived false}})
-(defmethod serdes/dependencies "Action" [action]
+(defmethod serdes/deserialization-dependencies "Action" [action]
(set
(concat
;; other stuff is implicitly referenced through a Card
@@ -467,7 +467,19 @@
(let [{:keys [database_id dataset_query]} (first (:query action))]
(concat
[[{:model "Database" :id database_id}]]
- (serdes/mbql-deps dataset_query)))))))
+ (serdes/mbql-deps false dataset_query)))))))
+
+(defmethod serdes/serialization-dependencies "Action" [_model-name {:keys [id model_id type]}]
+ ;; Serialization runs on the raw entity, whose query lives in the `query_action` child table (`:type` is a keyword
+ ;; here, not a string), so the query is fetched rather than read from a nested `:query` key.
+ (set
+ (concat
+ (when model_id [[{:model "Card" :id model_id}]])
+ (when (= type :query)
+ (when-let [{:keys [database_id dataset_query]} (t2/select-one :model/QueryAction :action_id id)]
+ (concat
+ (when database_id [[{:model "Database" :id database_id}]])
+ (serdes/mbql-deps true dataset_query)))))))
(defmethod serdes/storage-path "Action" [action _ctx]
[{:label "actions"} {:label (:name action) :key (:entity_id action)}])
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index e8e2c90d7b2..c466beafbb8 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -2107,11 +2107,18 @@
;; stable filename de-dup suffixes across exports, see GHY-3754
:order-by serdes/stable-storage-order}))))
-(defmethod serdes/dependencies "Collection"
+(defmethod serdes/deserialization-dependencies "Collection"
[{:keys [parent_id]}]
(when parent_id
#{[{:model "Collection" :id parent_id}]}))
+(defmethod serdes/serialization-dependencies "Collection"
+ ;; The raw entity carries its ancestry in `:location` (the ingested form is normalized to `:parent_id`), so the
+ ;; parent Collection is derived from the path. A selective export may legitimately omit the parent.
+ [_model-name {:keys [location]}]
+ (when-let [parent-id (location-path->parent-id location)]
+ #{[{:model "Collection" :id parent-id}]}))
+
(defmethod serdes/generate-path "Collection" [_ coll]
(serdes/maybe-labeled "Collection" coll :slug))
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index cbef429b98c..d9f59cf1c1f 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -445,21 +445,27 @@
:show_in_getting_started false}})
(defn- serdes-deps-dashcard
- [{:keys [action_id card_id parameter_mappings visualization_settings series]}]
+ [allow-int-ids? {:keys [action_id card_id parameter_mappings visualization_settings series]}]
(set
(concat
- (mapcat serdes/mbql-deps parameter_mappings)
- (serdes/visualization-settings-deps visualization_settings)
+ (mapcat #(serdes/mbql-deps allow-int-ids? %) parameter_mappings)
+ (serdes/visualization-settings-deps allow-int-ids? visualization_settings)
(when card_id #{[{:model "Card" :id card_id}]})
(when action_id #{[{:model "Action" :id action_id}]})
(for [s series] [{:model "Card" :id (:card_id s)}]))))
-(defmethod serdes/dependencies "Dashboard"
- [{:keys [collection_id dashcards parameters]}]
- (->> (map serdes-deps-dashcard dashcards)
+(defn- dashboard-deps
+ "The serdes dependencies of a Dashboard as `:serdes/meta` paths. Expects `:dashcards` (with `:series`) inlined — the
+ ingested shape, which the raw path hydrates first. `allow-int-ids?` selects raw-appdb vs serialized ref semantics. Shared by
+ [[serdes/deserialization-dependencies]] (allow-int-ids? false) and [[serdes/serialization-dependencies]] (allow-int-ids? true)."
+ [allow-int-ids? {:keys [collection_id dashcards parameters]}]
+ (->> (map #(serdes-deps-dashcard allow-int-ids? %) dashcards)
(reduce set/union #{})
(set/union (when collection_id #{[{:model "Collection" :id collection_id}]}))
- (set/union (serdes/parameters-deps parameters))))
+ (set/union (serdes/parameters-deps allow-int-ids? parameters))))
+
+(defmethod serdes/deserialization-dependencies "Dashboard" [dashboard]
+ (dashboard-deps false dashboard))
(defmethod serdes/descendants "Dashboard" [_model-name id _opts]
(let [dashcards (t2/select [:model/DashboardCard :id :card_id :action_id :parameter_mappings :visualization_settings]
@@ -493,6 +499,13 @@
(into {} (for [card-id (some->> dashboard :parameters (keep (comp :card_id :values_source_config)))]
{["Card" card-id] {"Dashboard" dash-id}})))))
+(defmethod serdes/serialization-dependencies "Dashboard" [_model-name dashboard]
+ ;; a raw Dashboard has its dashcards/series in separate tables; hydrate them into the inlined shape dashboard-deps
+ ;; expects, then reuse the same walk as deserialization.
+ (dashboard-deps true (-> dashboard
+ (t2/hydrate :dashcards)
+ (update :dashcards #(t2/hydrate % :series)))))
+
;;;; ------------------------------------------------- Search ----------------------------------------------------------
(search/define-spec "dashboard"
diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.clj
index ce19a582abd..25339e87376 100644
--- a/src/metabase/documents/models/document.clj
+++ b/src/metabase/documents/models/document.clj
@@ -245,12 +245,30 @@
:else
nil))))))
-(defmethod serdes/dependencies "Document"
+(defmethod serdes/deserialization-dependencies "Document"
[{:keys [collection_id] :as document}]
(set (concat
(document-deps document)
(when collection_id #{[{:model "Collection" :id collection_id}]}))))
+(defmethod serdes/serialization-dependencies "Document"
+ ;; Embedded cards and smart-links become content references (each must be part of the export); the containing
+ ;; Collection is included too, though a selective export may legitimately omit it.
+ [_model-name {:keys [collection_id content_type] :as document}]
+ (set
+ (concat
+ (when collection_id [[{:model "Collection" :id collection_id}]])
+ (when (= content_type prose-mirror/prose-mirror-content-type)
+ (concat
+ (for [embedded-card-id (prose-mirror/card-ids document)]
+ [{:model "Card" :id embedded-card-id}])
+ (for [{model :model link-id :entityId}
+ (prose-mirror/collect-ast document
+ #(when (= prose-mirror/smart-link-type (:type %))
+ (:attrs %)))
+ :when (contains? model->serdes-model model)]
+ [{:model (model->serdes-model model) :id link-id}]))))))
+
(defmethod serdes/descendants "Document"
[_model-name id _opts]
(when-let [document (t2/select-one :model/Document :id id)]
diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.clj
index bca7ecb20e0..0ad66be8a2d 100644
--- a/src/metabase/measures/models/measure.clj
+++ b/src/metabase/measures/models/measure.clj
@@ -190,8 +190,8 @@
[_measure]
[:name (serdes/hydrated-hash :table) :created_at])
-(defmethod serdes/dependencies "Measure" [{:keys [definition]}]
- (serdes/mbql-deps definition))
+(defmethod serdes/deserialization-dependencies "Measure" [{:keys [definition]}]
+ (serdes/mbql-deps false definition))
(defmethod serdes/storage-path "Measure" [measure _ctx]
(let [table-path (-> measure :definition serdes/serialized-query-source-table)]
diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.clj
index 49369214244..5b8b6740fee 100644
--- a/src/metabase/metabot/models/metabot.clj
+++ b/src/metabase/metabot/models/metabot.clj
@@ -41,9 +41,9 @@
[_table]
[:name])
-(defmethod serdes/dependencies "Metabot"
+(defmethod serdes/deserialization-dependencies "Metabot"
[{:keys [collection_id prompts]}]
- (cond-> (set (mapcat serdes/dependencies prompts))
+ (cond-> (set (mapcat serdes/deserialization-dependencies prompts))
collection_id (conj [{:model "Collection" :id collection_id}])))
(defmethod serdes/generate-path "Metabot" [_ metabot]
diff --git a/src/metabase/metabot/models/metabot_prompt.clj b/src/metabase/metabot/models/metabot_prompt.clj
index 2cdc0c12c99..f1f780d775a 100644
--- a/src/metabase/metabot/models/metabot_prompt.clj
+++ b/src/metabase/metabot/models/metabot_prompt.clj
@@ -28,7 +28,7 @@
(conj (serdes/generate-path "Metabot" (t2/select-one :model/Metabot (:metabot_id entity)))
(serdes/infer-self-path "MetabotPrompt" entity)))
-(defmethod serdes/dependencies "MetabotPrompt" [prompt]
+(defmethod serdes/deserialization-dependencies "MetabotPrompt" [prompt]
#{[{:model "Card" :id (:card_id prompt)}]})
(defmethod serdes/make-spec "MetabotPrompt" [_model-name _opts]
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 301a15d8c90..c99b5a8941f 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -24,7 +24,7 @@
tests)
- `serdes/descendants` - if model should be extracted along with some support (like Collection and all its
children), you need to specify what to fetch - see `Card` for an example (used during export)
- - `serdes/dependencies` - if model references other models, you need to declare which ones, see `Action` for an
+ - `serdes/deserialization-dependencies` - if model references other models, you need to declare which ones, see `Action` for an
example (used during import)
- Write tests
- basic layout (that all entity fields are mentioned, fks are marked as such) is tested automatically
@@ -86,6 +86,10 @@
;; there was no science behind picking 100 as a number
(def ^:private extract-nested-batch-limit "max amount of entities to fetch nested entities for" 100)
+(def query-batch-size
+ "Maximum number of ids per `:in` clause, to stay under database parameter limits."
+ 1000)
+
(mr/def ::model-keyword
[:and
qualified-keyword?
@@ -654,6 +658,19 @@
(defmethod required :default [_ _]
nil)
+(defmulti serialization-dependencies
+ "The references of a raw appdb `entity` (a Toucan instance, not a serialized map) as a seq of dependency maps
+ `{:model \"ModelName\", :id id}`. Its load-time counterpart is [[deserialization-dependencies]], which runs on the
+ serialized form and omits tables/fields (import creates them on the fly); this one reports them so the export can
+ existence-check them. Consumed by [[metabase-enterprise.serialization.v2.dependency-validation]].
+
+ Dispatched on model-name. Default returns `nil`, so only models with references need to implement this."
+ {:arglists '([model-name entity])}
+ (fn [model-name _entity] model-name))
+
+(defmethod serialization-dependencies :default [_ _]
+ nil)
+
;;; # Metadata Export
(defmulti metadata-query
@@ -713,7 +730,7 @@
;;; Then for each ingested entity:
;;;
;;; - `(ingest-one serdes-path opts)` is called to read the value into memory, then
-;;; - `(dependencies ingested)` gets a list of other `:serdes/meta` paths need to be loaded first.
+;;; - `(deserialization-dependencies ingested)` gets a list of other `:serdes/meta` paths need to be loaded first.
;;; - See below on dependencies.
;;; - Dependencies are loaded recursively in postorder; that is an entity is loaded after all its deps.
;;; - Circular dependencies will make the load process throw.
@@ -766,7 +783,7 @@
;;; ## Dependencies
;;; The files of an export are returned in arbitrary order by [[ingest-list]]. But in order to load any entity,
;;; everything it has a foreign key to must be loaded first. This is the purpose of one of the most complicated parts of
-;;; serdes: [[dependencies]].
+;;; serdes: [[deserialization-dependencies]].
;;;
;;; This multimethod returns a list (possibly empty) of `:serdes/meta` paths that this entity depends on. A `Card`
;;; depends on the `Table`s it queries, the `Collection` it belongs to, and possibly much else.
@@ -777,18 +794,19 @@
;;; Missing dependencies will cause flaky deserialization failures, since sometimes the FK target will exist already,
;;; and sometimes not, depending on the arbitrary order of `ingest-list`.
-(defmulti dependencies
+(defmulti deserialization-dependencies
"Given an entity map as ingested (not a Toucan entity) returns a (possibly empty) list of its dependencies, where each
dependency is represented by its abstract path (its `:serdes/meta` value).
- NOTE: This is called during **LOAD**.
+ NOTE: This is called during **LOAD**. Its export-time counterpart is [[serialization-dependencies]], which runs on a
+ raw entity and additionally reports tables/fields (which import synthesizes on the fly, so they aren't load deps).
Keyed on the model name for this entity.
Default implementation returns `nil`, so only models that have dependencies need to implement this."
{:arglists '([ingested])}
ingested-model)
-(defmethod dependencies :default [_]
+(defmethod deserialization-dependencies :default [_]
nil)
(defmulti load-update!
@@ -1030,7 +1048,7 @@
The identifier can be a single entity ID string, a single identity-hash string, or a vector of entity ID and hash
strings. If the ID is compound, then the last ID is the one that corresponds to the model. This allows for the
- compound IDs needed for nested entities like `DashboardCard`s to get their [[dependencies]].
+ compound IDs needed for nested entities like `DashboardCard`s to get their [[deserialization-dependencies]].
Throws if the corresponding entity cannot be found.
@@ -1111,7 +1129,7 @@
(defn table->path
"Given a `table_id` as exported by [[export-table-fk]], turn it into a `[{:model ...}]` path for the Table.
- This is useful for writing [[dependencies]] implementations."
+ This is useful for writing [[deserialization-dependencies]] implementations."
[[db-name schema table-name]]
(filterv some? [{:model "Database" :id db-name}
(when schema {:model "Schema" :id schema})
@@ -1198,7 +1216,7 @@
(defn field->path
"Given a `field_id` as exported by [[export-field-fk]], turn it into a `[{:model ...}]` path for the Field.
- This is useful for writing [[dependencies]] implementations."
+ This is useful for writing [[deserialization-dependencies]] implementations."
[[db-name schema table-name field-name]]
(filterv some? [{:model "Database" :id db-name}
(when schema {:model "Schema" :id schema})
@@ -1437,6 +1455,14 @@
(declare ^:private mbql-deps-map)
+(defn- raw-ref-id?
+ "True if `x` is a raw (numeric) reference id — i.e. a positive integer — and `allow-int-ids?` is set, meaning we're computing
+ *serialization* (export) deps from a raw entity. The `mbql-*` walkers serve both forms; a numeric is only treated as
+ a reference when `allow-int-ids?`, since a serialized query can still carry a stray numeric id (e.g. a query whose deleted
+ source database never got name-ified) that must not be mistaken for a live ref at load time."
+ [allow-int-ids? x]
+ (and allow-int-ids? (pos-int? x)))
+
(defn- ref->db-dep
"Given a portable table or field reference (a vector like `[db-name schema table-name ...]`), return a set with
its Database dependency, or nil. Table and Field references are intentionally *not* dependencies — missing ones
@@ -1447,49 +1473,68 @@
(when-let [db-name (first ref)]
#{[{:model "Database" :id db-name}]}))
-(defn- mbql-deps-vector [entity]
+(def ^:private mbql-ref-tag->model
+ "The serdes model that a `:metric`/`:segment`/`:measure` MBQL reference clause depends on."
+ {:metric "Card", "metric" "Card"
+ :segment "Segment", "segment" "Segment"
+ :measure "Measure", "measure" "Measure"})
+
+(defn- mbql-deps-vector [allow-int-ids? entity]
(match/match-one entity
- ;; A `:field`/`:field-id` clause's only dependency is the Database of its referenced field; the Field itself
- ;; is not a dependency, and a field clause never contains nested metric/segment/card refs, so we don't descend.
+ ;; --- serialized (portable) refs, walked at load time ---
+ ;; A serialized `:field` clause's only dependency is the Database of its referenced field; the Field/Table
+ ;; themselves are synthesized on import, and a field clause never nests metric/segment/card refs, so we don't
+ ;; descend.
[#{:field "field"} (_opts :guard map?) (ref :guard vector?)]
(ref->db-dep ref)
[(tag :guard #{:metric "metric" :segment "segment" :measure "measure"})
(opts :guard map?)
(field :guard portable-id?)]
- (into #{[{:model (case tag
- (:metric "metric") "Card"
- (:segment "segment") "Segment"
- (:measure "measure") "Measure")
- :id field}]}
- (mbql-deps-map opts))
-
- ;; legacy (MBQL 4) refs
+ (into #{[{:model (mbql-ref-tag->model tag) :id field}]}
+ (mbql-deps-map allow-int-ids? opts))
+
+ ;; legacy (MBQL 4) serialized refs
[#{:field "field" :field-id "field-id"} (ref :guard vector?) _opts]
(ref->db-dep ref)
[(tag :guard #{:metric "metric" :segment "segment" :measure "measure"}) (field :guard portable-id?)]
- #{[{:model (case tag
- (:metric "metric") "Card"
- (:segment "segment") "Segment"
- (:measure "measure") "Measure")
- :id field}]}
+ #{[{:model (mbql-ref-tag->model tag) :id field}]}
+
+ ;; --- raw (numeric) refs, walked at export time before serialization ---
+ ;; The referenced Field/Table are real appdb ids, existence-checked so a deleted row can't produce a malformed
+ ;; portable reference. `allow-int-ids?` gates these so a stray numeric in serialized data isn't mistaken for a ref.
+ [#{:field "field"} (_opts :guard map?) (id :guard (raw-ref-id? allow-int-ids? id))]
+ #{[{:model "Field" :id id}]}
+
+ [(tag :guard #{:metric "metric" :segment "segment" :measure "measure"})
+ (opts :guard map?)
+ (id :guard (raw-ref-id? allow-int-ids? id))]
+ (into #{[{:model (mbql-ref-tag->model tag) :id id}]}
+ (mbql-deps-map allow-int-ids? opts))
+
+ [#{:field "field" :field-id "field-id"} (id :guard (raw-ref-id? allow-int-ids? id)) _opts]
+ #{[{:model "Field" :id id}]}
+
+ [(tag :guard #{:metric "metric" :segment "segment" :measure "measure"}) (id :guard (raw-ref-id? allow-int-ids? id))]
+ #{[{:model (mbql-ref-tag->model tag) :id id}]}
_ (reduce #(cond
- (map? %2) (into %1 (mbql-deps-map %2))
- (vector? %2) (into %1 (mbql-deps-vector %2))
+ (map? %2) (into %1 (mbql-deps-map allow-int-ids? %2))
+ (vector? %2) (into %1 (mbql-deps-vector allow-int-ids? %2))
:else %1)
#{}
entity)))
-(defn- mbql-deps-map [m]
+(defn- mbql-deps-map [allow-int-ids? m]
(into #{}
(mapcat (fn [[k v]]
(cond
+ ;; --- serialized (portable) refs. Table/Field references contribute only their Database as a
+ ;; dependency (see `ref->db-dep`); the referenced Table/Field are synthesized on import. ---
(and (= k :database)
(string? v)
(not= v "database/__virtual")) #{[{:model "Database" :id v}]}
- ;; Table/Field references contribute only their Database as a dependency (see `ref->db-dep`).
(and (= k :source-table) (vector? v)) (ref->db-dep v)
(and (= k :source-table) (portable-id? v)) #{[{:model "Card" :id v}]}
(and (= k :source-card) (portable-id? v)) #{[{:model "Card" :id v}]}
@@ -1497,19 +1542,32 @@
(and (= k :snippet-id) (portable-id? v)) #{[{:model "NativeQuerySnippet" :id v}]}
(and (= k :table-id) (vector? v)) (ref->db-dep v)
(and (#{:card_id :card-id} k) (string? v)) #{[{:model "Card" :id v}]}
- (map? v) (mbql-deps-map v)
- (vector? v) (mbql-deps-vector v))))
+ ;; --- raw (numeric) refs, walked at export time: the referenced Table/Field are real appdb ids to
+ ;; existence-check. `allow-int-ids?` gates these (see `raw-ref-id?`). ---
+ (and (= k :database) (raw-ref-id? allow-int-ids? v)) #{[{:model "Database" :id v}]}
+ (and (= k :source-table) (raw-ref-id? allow-int-ids? v)) #{[{:model "Table" :id v}]}
+ (and (= k :source-card) (raw-ref-id? allow-int-ids? v)) #{[{:model "Card" :id v}]}
+ (and (= k :source-field) (raw-ref-id? allow-int-ids? v)) #{[{:model "Field" :id v}]}
+ (and (= k :snippet-id) (raw-ref-id? allow-int-ids? v)) #{[{:model "NativeQuerySnippet" :id v}]}
+ (and (= k :table-id) (raw-ref-id? allow-int-ids? v)) #{[{:model "Table" :id v}]}
+ (and (#{:card_id :card-id} k) (raw-ref-id? allow-int-ids? v)) #{[{:model "Card" :id v}]}
+ (map? v) (mbql-deps-map allow-int-ids? v)
+ (vector? v) (mbql-deps-vector allow-int-ids? v))))
m))
(defn mbql-deps
- "Given an MBQL expression as exported, with qualified names like `[\"some-db\" \"schema\" \"table_name\"]` instead of
- raw IDs, return the corresponding set of serdes dependencies. The query can't be imported until all the referenced
- databases, tables and fields are loaded."
- [entity]
+ "Given an MBQL expression, return the corresponding set of serdes dependencies. The query can't be imported until all
+ the referenced databases, tables and fields are loaded.
+
+ Pass `allow-int-ids?` true when `entity` is a raw appdb query whose references are still integer ids (export /
+ serialization time) and you want those references collected as dependencies. Pass false when `entity` is in
+ serialized/portable form (load time), where references are qualified names like `[\"some-db\" \"schema\"
+ \"table_name\"]` and a bare integer is a stray value to ignore, not a reference."
+ [allow-int-ids? entity]
(cond
- (map? entity) (mbql-deps-map entity)
- (seqable? entity) (mbql-deps-vector entity)
- :else (mbql-deps-vector [entity])))
+ (map? entity) (mbql-deps-map allow-int-ids? entity)
+ (seqable? entity) (mbql-deps-vector allow-int-ids? entity)
+ :else (mbql-deps-vector allow-int-ids? [entity])))
;;; ## Dashboard/Question Parameters
@@ -1559,14 +1617,14 @@
(defn parameters-deps
"Given the :parameters (possibly nil) for an entity, return any embedded serdes-deps as a set.
- Always returns an empty set even if the input is nil."
- [parameters]
+ Always returns an empty set even if the input is nil. For `allow-int-ids?` see [[mbql-deps]]."
+ [allow-int-ids? parameters]
(reduce set/union #{}
(for [parameter parameters
:when (= (keyword (:values_source_type parameter)) :card)
:let [config (:values_source_config parameter)]]
(set/union #{[{:model "Card" :id (:card_id config)}]}
- (mbql-deps-vector (:value_field config))))))
+ (mbql-deps-vector allow-int-ids? (:value_field config))))))
;;; ## Viz settings
@@ -1843,13 +1901,16 @@
(update :column_settings import-column-settings))))
(defn- viz-link-card-deps
- [settings]
+ [allow-int-ids? settings]
(when-let [{:keys [model id]} (get-in settings [:link :entity])]
- #{(case model
- ;; A linked Table is not a dependency (synthesized on import if missing), but its Database is.
- "table" [{:model "Database" :id (first id)}]
- [{:model (name (link-card-model->toucan-model model))
- :id id}])}))
+ (if (= model "table")
+ ;; Serialized: a linked Table is not a dependency (synthesized on import), but its Database is. Raw (export
+ ;; time): the numeric table id is a real Table to existence-check.
+ (cond
+ (vector? id) #{[{:model "Database" :id (first id)}]}
+ (raw-ref-id? allow-int-ids? id) #{[{:model "Table" :id id}]})
+ #{[{:model (name (link-card-model->toucan-model model))
+ :id id}]})))
(defn- viz-click-behavior-deps
[settings]
@@ -1865,21 +1926,21 @@
(defn visualization-settings-deps
"Given the :visualization_settings (possibly nil) for an entity, return any embedded serdes-deps as a set.
- Always returns an empty set even if the input is nil."
- [viz]
+ Always returns an empty set even if the input is nil. For `allow-int-ids?` see [[mbql-deps]]."
+ [allow-int-ids? viz]
(let [column-settings-keys-deps (some->> viz
:column_settings
keys
- (map (comp mbql-deps json/decode name)))
+ (map (comp #(mbql-deps allow-int-ids? %) json/decode name)))
column-settings-vals-deps (some->> viz
:column_settings
vals
(map viz-click-behavior-deps))
- link-card-deps (viz-link-card-deps viz)
+ link-card-deps (viz-link-card-deps allow-int-ids? viz)
click-behavior-deps (viz-click-behavior-deps viz)]
(->> (concat column-settings-keys-deps
column-settings-vals-deps
- [(mbql-deps viz) link-card-deps click-behavior-deps])
+ [(mbql-deps allow-int-ids? viz) link-card-deps click-behavior-deps])
(filter some?)
(reduce set/union #{}))))
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index fa0f1d31f48..b58d84b428b 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -190,11 +190,17 @@
(when-let [collection_id (t2/select-one-fn :collection_id :model/NativeQuerySnippet :id id)]
{["Collection" collection_id] {"NativeQuerySnippet" id}}))
-(defmethod serdes/dependencies "NativeQuerySnippet"
+(defmethod serdes/deserialization-dependencies "NativeQuerySnippet"
[{:keys [collection_id]}]
(when collection_id
[[{:model "Collection" :id collection_id}]]))
+(defmethod serdes/serialization-dependencies "NativeQuerySnippet"
+ [_model-name {:keys [collection_id]}]
+ ;; A snippet only references its containing Collection, which a selective export may legitimately omit.
+ (when collection_id
+ #{[{:model "Collection" :id collection_id}]}))
+
(defmethod serdes/storage-path "NativeQuerySnippet" [snippet ctx]
(serdes/storage-default-collection-path snippet ctx "snippets"))
diff --git a/src/metabase/osi/models/osi_ai_context.clj b/src/metabase/osi/models/osi_ai_context.clj
index 64fa6951526..ca76e0bfbf0 100644
--- a/src/metabase/osi/models/osi_ai_context.clj
+++ b/src/metabase/osi/models/osi_ai_context.clj
@@ -159,7 +159,7 @@
:import-with-context
(fn [current _ _] (:entity_local_id (parent-path->entity (pop (serdes/path current)))))}}})
-(defmethod serdes/dependencies "OsiAiContext"
+(defmethod serdes/deserialization-dependencies "OsiAiContext"
[entity]
;; Depend on the entity this row describes (its parent path), so it imports after that entity exists.
[(vec (pop (serdes/path entity)))])
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index a9ed68c343e..3c627d2ff7e 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1343,10 +1343,10 @@
;; FIXME: remove that `if` after v52
(m/update-existing :fk_target_field_id #(if (number? %) % (serdes/*import-field-fk* %)))))))
-(defn- result-metadata-deps [metadata]
+(defn- result-metadata-deps [allow-int-ids? metadata]
(when (seq metadata)
(-> (reduce into #{} (for [m metadata]
- (serdes/mbql-deps (:field_ref m))))
+ (serdes/mbql-deps allow-int-ids? (:field_ref m))))
(disj nil))))
(defmethod serdes/storage-path "Card" [card ctx]
@@ -1418,24 +1418,28 @@
:collection_preview true
:enable_embedding false}})
-(defmethod serdes/dependencies "Card"
- [{:keys [collection_id database_id dataset_query parameters parameter_mappings
- result_metadata source_card_id visualization_settings
- dashboard_id document_id]}]
+(defn- card-deps
+ "The serdes dependencies of a Card as `:serdes/meta` paths. `allow-int-ids?` selects raw-appdb vs serialized ref semantics for
+ the mbql walkers. Shared by [[serdes/deserialization-dependencies]] (allow-int-ids? false) and
+ [[serdes/serialization-dependencies]] (allow-int-ids? true)."
+ [allow-int-ids? {:keys [collection_id database_id dataset_query parameters parameter_mappings
+ result_metadata source_card_id visualization_settings
+ dashboard_id document_id]}]
(set
(concat
- (mapcat serdes/mbql-deps parameter_mappings)
- (serdes/parameters-deps parameters)
+ (mapcat #(serdes/mbql-deps allow-int-ids? %) parameter_mappings)
+ (serdes/parameters-deps allow-int-ids? parameters)
(when database_id [[{:model "Database" :id database_id}]])
- ;; Note: `table_id` is intentionally not a dependency — the Database (above) is, and a missing Table is
- ;; synthesized as an inactive row on import.
(when source_card_id #{[{:model "Card" :id source_card_id}]})
(when collection_id #{[{:model "Collection" :id collection_id}]})
(when dashboard_id #{[{:model "Dashboard" :id dashboard_id}]})
(when document_id #{[{:model "Document" :id document_id}]})
- (result-metadata-deps result_metadata)
- (serdes/mbql-deps dataset_query)
- (serdes/visualization-settings-deps visualization_settings))))
+ (result-metadata-deps allow-int-ids? result_metadata)
+ (serdes/mbql-deps allow-int-ids? dataset_query)
+ (serdes/visualization-settings-deps allow-int-ids? visualization_settings))))
+
+(defmethod serdes/deserialization-dependencies "Card" [card]
+ (card-deps false card))
(defmethod serdes/descendants "Card" [_model-name id _opts]
(let [card (t2/select-one :model/Card :id id)
@@ -1455,6 +1459,9 @@
(for [snippet-id snippets]
{["NativeQuerySnippet" snippet-id] {"Card" id}})))))
+(defmethod serdes/serialization-dependencies "Card" [_model-name card]
+ (card-deps true card))
+
;;;; ------------------------------------------------- Search ----------------------------------------------------------
(mu/defn- dataset-query->dimensions :- [:maybe [:sequential ::lib.schema.metadata/column]]
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index 431ffc41cde..66e34e9282e 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -217,8 +217,8 @@
[_segment]
[:name (serdes/hydrated-hash :table) :created_at])
-(defmethod serdes/dependencies "Segment" [{:keys [definition]}]
- (serdes/mbql-deps definition))
+(defmethod serdes/deserialization-dependencies "Segment" [{:keys [definition]}]
+ (serdes/mbql-deps false definition))
(defmethod serdes/storage-path "Segment" [segment _ctx]
(let [table-path (-> segment :definition serdes/serialized-query-source-table)]
diff --git a/src/metabase/timeline/models/timeline.clj b/src/metabase/timeline/models/timeline.clj
index e3a79a99cba..ee67759a71a 100644
--- a/src/metabase/timeline/models/timeline.clj
+++ b/src/metabase/timeline/models/timeline.clj
@@ -52,9 +52,14 @@
[_timeline]
[:name (serdes/hydrated-hash :collection) :created_at])
-(defmethod serdes/dependencies "Timeline" [{:keys [collection_id]}]
+(defmethod serdes/deserialization-dependencies "Timeline" [{:keys [collection_id]}]
[[{:model "Collection" :id collection_id}]])
+(defmethod serdes/serialization-dependencies "Timeline" [_model-name {:keys [collection_id]}]
+ ;; A Timeline only references its containing Collection, which a selective export may legitimately omit.
+ (when collection_id
+ #{[{:model "Collection" :id collection_id}]}))
+
(defmethod serdes/make-spec "Timeline" [_model-name opts]
{:copy [:archived :default :description :entity_id :icon :name]
:skip []
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 0fd4827321b..5ad30e73ea8 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -470,7 +470,7 @@
:import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id (merge {:sort-by (juxt :position :created_at)} opts))}})
-(defmethod serdes/dependencies "Transform"
+(defmethod serdes/deserialization-dependencies "Transform"
[{:keys [collection_id source tags source_database_id]}]
(set
(concat
@@ -480,7 +480,7 @@
[[{:model "Database" :id source_database_id}]])
(for [{tag-id :tag_id} tags]
[{:model "TransformTag" :id tag-id}])
- (serdes/mbql-deps source))))
+ (serdes/mbql-deps false source))))
(defmethod serdes/storage-path "Transform" [transform ctx]
(serdes/storage-default-collection-path transform ctx "transforms"))
diff --git a/src/metabase/transforms/models/transform_job.clj b/src/metabase/transforms/models/transform_job.clj
index cd41a06a274..ba544235302 100644
--- a/src/metabase/transforms/models/transform_job.clj
+++ b/src/metabase/transforms/models/transform_job.clj
@@ -222,7 +222,7 @@
:created_at (serdes/date)
:job_tags (serdes/nested :model/TransformJobTransformTag :job_id (merge {:sort-by (juxt :position :created_at)} opts))}})
-(defmethod serdes/dependencies "TransformJob"
+(defmethod serdes/deserialization-dependencies "TransformJob"
[{:keys [job_tags]}]
(set
(for [{tag-id :tag_id} job_tags]
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 7c6a46cbbd7..69f2f7739e3 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -466,7 +466,7 @@
field-q (serdes/recursively-find-field-q (:id table) (map :id (reverse fields)))]
(t2/select-one :model/Field field-q)))
-(defmethod serdes/dependencies "Field" [field]
+(defmethod serdes/deserialization-dependencies "Field" [field]
(let [db-path (first (serdes/path field))]
#{[db-path]}))
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 08bb3a53ee6..48068517f31 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -44,7 +44,7 @@
(conj (serdes/generate-path "Field" {:id field_id})
{:model "FieldUserSettings" :id "1"}))
-(defmethod serdes/dependencies "FieldUserSettings" [fv]
+(defmethod serdes/deserialization-dependencies "FieldUserSettings" [fv]
(let [db-path (first (serdes/path fv))]
[[db-path]]))
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 39e3c2c0380..85827de54c7 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -559,7 +559,7 @@
(conj (serdes/generate-path "Field" {:id field_id})
{:model "FieldValues" :id "0"}))
-(defmethod serdes/dependencies "FieldValues" [fv]
+(defmethod serdes/deserialization-dependencies "FieldValues" [fv]
(let [db-path (first (serdes/path fv))]
[[db-path]]))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index a264ea6d46a..3079c618929 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -583,7 +583,7 @@
(t2/select-one :model/Database :id (:db_id table)))
;;; ------------------------------------------------- Serialization -------------------------------------------------
-(defmethod serdes/dependencies "Table" [{:keys [db_id collection_id transform_id]}]
+(defmethod serdes/deserialization-dependencies "Table" [{:keys [db_id collection_id transform_id]}]
(cond-> [[{:model "Database" :id db_id}]]
collection_id (conj [{:model "Collection" :id collection_id}])
transform_id (conj [{:model "Transform" :id transform_id}])))
diff --git a/test/metabase/documents/models/document_test.clj b/test/metabase/documents/models/document_test.clj
index 75da7c6d203..6e882b19e30 100644
--- a/test/metabase/documents/models/document_test.clj
+++ b/test/metabase/documents/models/document_test.clj
@@ -448,7 +448,7 @@
(let [document {:collection_id 123
:creator_id 456
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (= #{[{:model "Collection" :id 123}]}
deps))))))
@@ -463,7 +463,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 789}]))
(is (contains? deps [{:model "Collection" :id 123}])))))
@@ -484,7 +484,7 @@
:model "table"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 789}]))
(is (contains? deps [{:model "Dashboard" :id 456}]))
(is (contains? deps [{:model "Table" :id 321}]))
@@ -505,7 +505,7 @@
:model "card"}}]}]}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 999}]))
(is (contains? deps [{:model "Collection" :id 123}])))))
@@ -518,7 +518,7 @@
:content [{:type "text" :text "Plain text only"}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (= #{[{:model "Collection" :id 123}]}
deps)))))
@@ -536,7 +536,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 456}]))
(is (not (some #(= (:model (first %)) "unknown-model") deps))))))
@@ -553,7 +553,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 456}]))
(is (= 2 (count deps)))))) ; collection and one valid card
@@ -571,7 +571,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 789}]))
(is (= 2 (count deps)))))) ; collection and one unique card
@@ -583,7 +583,7 @@
:content []}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (= #{[{:model "Collection" :id 123}]}
deps)))))
@@ -600,7 +600,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 456}]))
(is (= 2 (count deps)))))) ; collection and one valid card
@@ -617,7 +617,7 @@
:model "card"}}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 456}]))
(is (= 2 (count deps)))))) ; collection and one valid card
@@ -647,7 +647,7 @@
:model "table"}}]}]}]}]}]}
:content_type "application/json+vnd.prose-mirror"
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Card" :id 789}]))
(is (contains? deps [{:model "Dashboard" :id 456}]))
(is (contains? deps [{:model "Table" :id 321}]))
@@ -664,7 +664,7 @@
:model "card"}}]}
:content_type "application/json" ; Not prose-mirror
:serdes/meta [{:model "Document" :id "test-entity-id"}]}
- deps (serdes/dependencies document)]
+ deps (serdes/deserialization-dependencies document)]
(is (contains? deps [{:model "Collection" :id 123}])))))
(deftest document-serdes-descendants-embedded-cards-test
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index b574e15f6d6..39d1cd775cd 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -265,7 +265,36 @@
(testing "template tag :table-id contributes only its Database dependency — the referenced Table itself is not a
dependency (it's synthesized on import if missing)"
(is (= #{[{:model "Database" :id "DB"}]}
- (#'serdes/mbql-deps-map {:table-id ["DB" "SCHEMA" "TABLE"]})))))
+ (#'serdes/mbql-deps-map false {:table-id ["DB" "SCHEMA" "TABLE"]})))))
+
+(deftest ^:parallel mbql-deps-format-parity-test
+ (testing "mbql-deps finds each reference on both the serialized (portable) and the raw (numeric) form of a query.
+ serialization-dependencies runs on raw entities and existence-checks the referenced Table/Field;
+ deserialization-dependencies runs on the serialized form, where Table/Field are synthesized on import, so
+ it reports only their Database. Every other ref type resolves to the same model in both forms. This is the
+ parity guard for the two dependency codepaths sharing mbql-deps."
+ (let [models (fn [deps] (into #{} (map (comp :model last)) deps))
+ eid (fn [c] (apply str (repeat 21 c)))]
+ (testing "MBQL ref clauses"
+ (doseq [[label serialized raw ser-models raw-models]
+ [["field (pMBQL)" [:field {} ["DB" "S" "T" "F"]] [:field {} 53] #{"Database"} #{"Field"}]
+ ["field (legacy)" [:field ["DB" "S" "T" "F"] {}] [:field 53 {}] #{"Database"} #{"Field"}]
+ ["metric" [:metric {} (eid \a)] [:metric {} 99] #{"Card"} #{"Card"}]
+ ["segment" [:segment {} (eid \b)] [:segment {} 5] #{"Segment"} #{"Segment"}]
+ ["measure" [:measure {} (eid \c)] [:measure {} 3] #{"Measure"} #{"Measure"}]]]
+ (testing label
+ (is (= ser-models (models (serdes/mbql-deps false serialized))) "serialized (portable) form")
+ (is (= raw-models (models (serdes/mbql-deps true raw)))
+ "raw (numeric) form"))))
+ (testing "MBQL map keys"
+ (doseq [[label serialized raw ser-models raw-models]
+ [["source-table" {:source-table ["DB" "S" "T"]} {:source-table 9} #{"Database"} #{"Table"}]
+ ["source-card" {:source-card (eid \d)} {:source-card 7} #{"Card"} #{"Card"}]
+ ["snippet-id" {:snippet-id (eid \e)} {:snippet-id 2} #{"NativeQuerySnippet"} #{"NativeQuerySnippet"}]]]
+ (testing label
+ (is (= ser-models (models (#'serdes/mbql-deps-map false serialized))) "serialized")
+ (is (= raw-models (models (#'serdes/mbql-deps-map true raw)))
+ "raw")))))))
(deftest ^:parallel export-parameters-test
(binding [serdes/*export-fk* (fn [id model]