The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Validate serialization dependencies before export (GHY-4010) (#76763)
Eric Normandff5e3b6973ce
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.cljindex 75b952dbc0d..6a6d3d14445 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj@@ -116,7 +116,7 @@(try(zipmap (or (get @dep-cache dep)(get (swap! dep-cache assoc dep- (serdes/dependencies+ (serdes/deserialization-dependencies(serialization/ingest-one ingestable dep)))dep))(repeat dep))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.cljnew file mode 100644index 00000000000..facc4d9f1a5--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj@@ -0,0 +1,133 @@+(ns metabase-enterprise.serialization.v2.dependency-validation+ "Export-time dependency validation for serialization.++ Before an export is written, every entity that will be extracted is checked to make sure all of its references are+ satisfiable in the resulting archive: content references (Cards, Dashboards, …) must themselves be part of the+ export, and data-model references (Database, Table, Field) must exist in the source appdb. An unsatisfied reference+ would otherwise become a dangling reference or a malformed portable id in the archive, breaking the import.++ Historically this was the collection-id, Card-only \"escape analysis\"; it is now a general dependency-satisfaction+ check driven by [[metabase.models.serialization/serialization-dependencies]]."+ (:require+ [clojure.string :as str]+ [medley.core :as m]+ [metabase-enterprise.serialization.v2.models :as serdes.models]+ [metabase.models.serialization :as serdes]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- collection-label [coll-id]+ (if coll-id+ (let [collection (t2/hydrate (t2/select-one :model/Collection :id coll-id) :ancestors)+ names (->> (conj (:ancestors collection) collection)+ (map :name)+ (str/join " > "))]+ (format "%d: %s" coll-id names))+ "[no collection]"))++(defn- entity-label [{:keys [model id]}]+ (let [entity (t2/select-one [model :collection_id :name] :id id)]+ (format "%s %d (%s from collection %s)" (name model) id (:name entity) (collection-label (:collection_id entity)))))++(defn- resize-batch+ [[model batch]]+ (for [batch (partition-all serdes/query-batch-size batch)]+ [model batch]))++(defn- entity-deps+ "The dependency contribution of a single entity.++ Returns a `{:visited :deps}`. `serialization-dependencies` yields `:serdes/meta` paths (each a vector whose last+ element identifies the referenced entity), so we take that last element and tag it with the `:via` entity."+ [entity]+ (let [model (name (t2/model entity))+ via [model (:id entity)]]+ {:visited #{via}+ :deps (into #{}+ (map #(assoc (peek %) :via via))+ (serdes/serialization-dependencies model entity))}))++(defn- merge-entity-deps+ "Monoid merging [[entity-deps]] contributions: unions the `:visited` and `:deps` sets. Identity is empty sets."+ ([] {:visited #{} :deps #{}})+ ([acc] acc)+ ([acc contribution] (merge-with into acc contribution)))++(defn- collect-dependencies+ "Take the model/ids and return {:visited :deps}. Entities are streamed in batches, not held in memory."+ [by-model coll-set opts]+ (let [content-models (set serdes.models/content)]+ (transduce+ (comp+ (filter (comp content-models key))+ (mapcat resize-batch)+ (mapcat (fn [[model batch]]+ (serdes/extract-query model (merge opts {:collection-set coll-set+ :where [:in :id batch]}))))+ (map entity-deps))+ merge-entity-deps+ by-model)))++(defn- existing-ids+ "The subset of `ids` that exist as rows of `model` (a model-name string), queried in bounded `:in` batches."+ [model ids]+ (into #{}+ (comp (distinct)+ (partition-all serdes/query-batch-size)+ (mapcat #(t2/select-pks-set (keyword "model" model) {:where [:in :id %]})))+ ids))++(def ^:private structural-content-models+ "Content models whose absence from the archive is tolerated on import, so a reference to one is never a completeness+ failure. A selective export routinely omits the Collection tree an entity lives in; the entity then loads at the+ root of the target rather than producing a dangling reference."+ #{"Collection"})++(defn- unsatisfied-dependencies+ "The `deps` that won't be satisfied in the archive, each tagged with a `:reason` (see the namespace docstring for the+ data-model vs content classification). Two subtleties are encoded below: a content reference to a *deleted* entity+ is not a failure (export can't emit a portable id for a gone row, so `fk-elide` just drops it), and+ [[structural-content-models]] references are ignored entirely."+ [deps visited analytics-cards]+ (let [content-models (set serdes.models/content)+ {content-deps true data-deps false} (group-by #(contains? content-models (:model %)) deps)+ content-deps (remove (comp structural-content-models :model) content-deps)+ existing-data (m/map-kv-vals existing-ids (u/group-by :model :id data-deps))+ existing-content (m/map-kv-vals existing-ids (u/group-by :model :id content-deps))]+ (concat+ (for [{:keys [model id] :as dep} content-deps+ :when (and (contains? (get existing-content model) id)+ (not (contains? visited [model id]))+ (not (and (= model "Card") (contains? analytics-cards id))))]+ (assoc dep :reason :not-in-export))+ (for [{:keys [model id] :as dep} data-deps+ :when (not (contains? (get existing-data model) id))]+ (assoc dep :reason :missing-in-appdb)))))++(defn- unsatisfied-label [{:keys [model id via reason]}]+ (format "%s references %s %s which %s"+ (entity-label {:model (keyword "model" (first via)) :id (second via)})+ model id+ (case reason+ :not-in-export "is not included in the export"+ :missing-in-appdb "is missing from the source database")))++(defn validate-dependencies!+ "Validates that every entity to be extracted has all of its references satisfied (see the namespace docstring). Logs+ a warning per unsatisfied reference and, unless `:continue-on-error`, throws to abort the export before any archive+ is produced. `by-model` is a map of `{model-name [ids ...]}`."+ [by-model coll-set analytics-cards opts]+ (let [{:keys [visited deps]} (collect-dependencies by-model coll-set opts)+ missing (unsatisfied-dependencies deps visited analytics-cards)]+ (when (seq missing)+ (doseq [dep missing]+ (log/warnf "Failed to export: %s" (unsatisfied-label dep)))+ (when-not (:continue-on-error opts)+ (throw (ex-info (format (str "Serialization failed: %d reference(s) could not be satisfied, which would "+ "produce an incomplete export. See the warnings above for the affected "+ "entities. Pass continue-on-error to export anyway, skipping them.")+ (count missing))+ {:status-code 400}))))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 97fbe7c828e..b33d51f78d2 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -4,8 +4,8 @@See the detailed descriptions of the (de)serialization processes in [[metabase.models.serialization]]."(:require[clojure.set :as set]- [clojure.string :as str][metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]+ [metabase-enterprise.serialization.v2.dependency-validation :as dependency-validation][metabase-enterprise.serialization.v2.models :as serdes.models][metabase.collections.models.collection :as collection][metabase.models.serialization :as serdes]@@ -71,19 +71,6 @@(into (map :id) roots)(into (mapcat collection/descendant-ids) roots))))-(defn- collection-label [coll-id]- (if coll-id- (let [collection (t2/hydrate (t2/select-one :model/Collection :id coll-id) :ancestors)- names (->> (conj (:ancestors collection) collection)- (map :name)- (str/join " > "))]- (format "%d: %s" coll-id names))- "[no collection]"))--(defn- entity-label [{:keys [model id]}]- (let [entity (t2/select-one [model :collection_id :name] :id id)]- (format "%s %d (%s from collection %s)" (name model) id (:name entity) (collection-label (:collection_id entity)))))-(defn- parse-target [[model-name id :as target]](if (string? id)(if-let [resolved-id (serdes/eid->id model-name id)]@@ -103,68 +90,21 @@(mapcat collection/descendant-ids)analytics-roots)))-(defn- escape-analysis- "Analyzes the dependency graph to find cards that are outside the collection set (escapees).- Returns a map with:- - :reportable-escaped - escapees that should trigger warnings (non-analytics)- - :analytics-card-ids - card IDs in analytics collections (should be removed from extraction but not block export)-- Cards that depend on analytics cards are allowed to be exported - the references will be converted- to entity_ids during export and resolved on import since analytics cards have stable entity_ids."- [{colls "Collection" cards "Card" :as _by-model} nodes]- (log/tracef "Running escape analysis for %d colls and %d cards" (count colls) (count cards))- (when-let [colls (-> colls set not-empty)]- (let [clause {:where [:or- [:in :collection_id colls]- (when (contains? colls nil)- [:= :collection_id nil])]}- possible-pks (t2/select-pks-set :model/Card clause)- escaped-card-ids (set/difference (set cards) possible-pks)]- (when (seq escaped-card-ids)- ;; Analytics cards have stable entity_ids across instances, so cards that depend on them- ;; can still be exported: the references will be resolved on import- (let [analytics-colls (analytics-collection-ids)- escaped-in-analytics (if (seq analytics-colls)- (t2/select-pks-set :model/Card {:where [:and- [:in :id escaped-card-ids]- [:in :collection_id analytics-colls]]})- #{})- reportable-escaped (set/difference escaped-card-ids escaped-in-analytics)]- {:reportable-escaped (->> reportable-escaped- (mapv (fn [id]- (-> (get nodes ["Card" id])- (assoc :escapee {:model :model/Card- :id id})))))- :analytics-card-ids escaped-in-analytics})))))--(defn- handle-escapees!- "Reports cards that are referenced from inside the requested collections but saved outside them.-- Logs a per-entity warning for each escapee.-- Throws if `continue-on-error?` is false."- [escaped continue-on-error?]- (let [dashboards (group-by #(get % "Dashboard") escaped)]- (doseq [[dash-id escapes] (dissoc dashboards nil)]- (log/warnf "Failed to export Dashboard %d (%s) containing Card saved outside requested collections: %s"- dash-id- (t2/select-one-fn :name :model/Dashboard :id dash-id)- (str/join ", " (map #(entity-label (:escapee %)) escapes))))- (when-let [other (not-empty (get dashboards nil))]- (log/warnf "Failed to export Cards based on questions outside requested collections: %s"- (str/join ", " (for [item other]- (format "%s -> %s"- (if (get item "Card")- (entity-label {:model :model/Card :id (get item "Card")})- (dissoc item :escapee))- (entity-label (:escapee item))))))))- (when-not continue-on-error?- (throw (ex-info (format (str "Serialization failed: %d card(s) referenced by the requested collections "- "are saved outside them, which would produce an incomplete export. See the "- "warnings above for the affected entities. Pass continue-on-error to export "- "anyway, skipping the affected dashboards and cards.")- (count (distinct (map (comp :id :escapee) escaped))))- {:status-code 400}))))+(defn- analytics-card-ids+ "Of `card-ids`, the subset living in an 'analytics' namespace collection. These are removed from extraction (they+ have stable entity_ids across instances) and references to them stay valid, resolving on import. `card-ids` is+ queried in bounded `:in` batches so a large closure doesn't blow past database parameter limits."+ [card-ids]+ (let [analytics-colls (analytics-collection-ids)]+ (if (and (seq card-ids) (seq analytics-colls))+ (into #{}+ (comp (partition-all serdes/query-batch-size)+ (mapcat (fn [batch]+ (t2/select-pks-set :model/Card {:where [:and+ [:in :id (vec batch)]+ [:in :collection_id (vec analytics-colls)]]}))))+ card-ids)+ #{})))(defn- resolve-targets"Returns all targets (for either supplied initial `targets` or for supplied `user-id`)."@@ -200,16 +140,15 @@(resolve-targets opts user-id));; `by-model` is a map of `{model-name [ids ...]}`by-model (u/group-by first second (keys nodes))- ;; Escape analysis only matters when exporting collection content — it checks whether- ;; cards referenced by dashboards live outside the target collections.- {:keys [reportable-escaped analytics-card-ids]} (when has-content?- (escape-analysis by-model nodes))]- ;; A card referenced from inside the requested collections but living outside them would produce an incomplete- ;; export.- (when (seq reportable-escaped)- (handle-escapees! reportable-escaped (:continue-on-error opts))) ;; may throw- (let [coll-set (get by-model "Collection")- ;; When targets are specified, also include Tables found via descendants+ ;; Cards in analytics collections aren't extracted (stable entity_ids), but references to them stay valid.+ analytics-cards (when has-content?+ (analytics-card-ids (get by-model "Card")))+ coll-set (get by-model "Collection")]+ ;; Validate that every entity to be extracted has all its references satisfied; aborts (unless+ ;; continue-on-error) before any archive is produced, so incomplete exports fail fast and loudly.+ (when has-content?+ (dependency-validation/validate-dependencies! by-model coll-set analytics-cards opts)) ;; may throw+ (let [;; When targets are specified, also include Tables found via descendants;; (published tables in target collections). These are extracted by ID, not all.targeted-data-model (when (seq targets)(select-keys by-model serdes.models/data-model-in-collection))@@ -218,8 +157,8 @@(seq targeted-data-model) (merge targeted-data-model);; Remove analytics cards from extraction - they have stable entity_ids across instances;; so cards that reference them can still be exported and imported correctly- (and analytics-card-ids (contains? by-model "Card"))- (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))+ (and analytics-cards (contains? by-model "Card"))+ (update "Card" (fn [ids] (vec (remove analytics-cards ids)))))extract-by-ids (fn [[model ids]](serdes/extract-all model (merge opts {:collection-set coll-set:where [:in :id ids]})))@@ -243,6 +182,8 @@(merge(u/traverse colls #(serdes/ascendants (first %) (second %)))(u/traverse colls #(serdes/descendants (first %) (second %) {})))))- (def escaped (escape-analysis (u/group-by first second (keys nodes)) nodes))- ;; continue-on-error? true so this just logs in the REPL instead of throwing- (handle-escapees! (:reportable-escaped escaped) true))+ (def by-model (u/group-by first second (keys nodes)))+ ;; continue-on-error so this just logs in the REPL instead of throwing+ (dependency-validation/validate-dependencies! by-model (get by-model "Collection")+ (analytics-card-ids (get by-model "Card"))+ {:continue-on-error true}))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex ca6642beb17..d9cad571a2e 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -172,7 +172,7 @@;; `::strip` is handled in `metabase.serialization`(circular path) (assoc ::serdes/strip (keys-to-strip ingested)));; we need less deps when trying to load "stripped" data- deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))+ deps (->> (serdes/deserialization-dependencies (apply dissoc ingested (::serdes/strip ingested)))(remove seen))_ (when (seq deps)(log/debug "Loading dependencies"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex b7d5e225c67..9a2db8279bf 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -239,11 +239,11 @@(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))(deftest export-log-captures-extract-warnings-test- (testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)"- ;; A dashboard in the exported collection references a card living in a different collection. Escape analysis- ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. Under- ;; continue-on-error the export still completes, and that warning must land in export.log even though extract- ;; happens outside the storage logging block.+ (testing "export.log captures dependency-validation warnings emitted during the eager extract phase (GHY-3802)"+ ;; A dashboard in the exported collection references a card living in a different collection. Dependency+ ;; validation runs eagerly inside extract/extract (before storage streaming) and warns about the unsatisfied+ ;; reference. Under continue-on-error the export still completes, and that warning must land in export.log even+ ;; though extract happens outside the storage logging block.(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Collection target {:name "Target Collection"}:model/Collection other {:name "Other Collection"}@@ -256,8 +256,8 @@:continue_on_error true))log (read-export-log res)](is (some? log) "export.log should be present in the archive")- (is (re-find #"outside requested collections" log)- "export.log should contain the escape-analysis warning emitted during extract"))))))+ (is (re-find #"not included in the export" log)+ "export.log should contain the dependency-validation warning emitted during extract"))))))(deftest export-aborts-on-escaped-card-test(testing "Export fails loudly with a 4xx instead of silently emitting an empty archive when a referenced card lives outside the requested collections (#75176)"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.cljnew file mode 100644index 00000000000..d9c53a56454--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/dependency_validation_test.clj@@ -0,0 +1,254 @@+(ns metabase-enterprise.serialization.v2.dependency-validation-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.test-util :as ts]+ [metabase-enterprise.serialization.v2.extract :as extract]+ [metabase.documents.prose-mirror :as prose-mirror]+ [metabase.models.serialization :as serdes]+ [metabase.search.test-util :as search.tu]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(use-fixtures :each (fn [f] (search.tu/with-index-disabled (f))))++(defn- by-model [model-name extraction]+ (->> extraction+ (into [])+ (filter #(= model-name ((comp :model last :serdes/meta) %)))))++(defn- ids-by-model [model-name extraction]+ (->> (by-model model-name extraction)+ (map (comp :id last :serdes/meta))+ set))++(defn- extract-aborts!+ "Realize `(extract/extract opts)`, asserting it aborts because a reference can't be satisfied (#75176, GHY-4010).+ Warnings are logged before the abort, so callers can still assert on the logged messages."+ [opts]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"incomplete export"+ (into [] (extract/extract opts)))))++(deftest escape-report-test+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}+ :model/Collection {coll2-id :id} {:name "Other Collection"}+ :model/Collection {coll3-id :id} {:name "Third Collection"}+ :model/Collection {clean-coll-id :id+ clean-coll-eid :entity_id} {:name "Clean Collection"}+ :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}+ ;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}+ :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}+ :model/Card {clean-card-eid :entity_id} {:name "Clean Card"+ :collection_id clean-coll-id+ :database_id db-id}+ :model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}+ :model/Card _ {:name "Dependent Card"+ :collection_id coll2-id+ :dataset_query {:type :query+ :database db-id+ :query {:source-table (str "card__" card1-id)}}}+ :model/User user {:email "[email protected]"}+ :model/Collection pcoll {:name "Personal Collection"+ :personal_owner_id (:id user)}+ :model/Card pcard {:name "Personal Card"+ :collection_id (:id pcoll)}+ :model/Card _ {:name "External Card"+ :dataset_query {:database db-id+ :type :query+ :query {:source-table (str "card__" (:id pcard))}}}+ :model/Card _ {:name "Card with parameters"+ :collection_id coll3-id+ :parameters [{:id "abc"+ :type "category"+ :values_source_type "card"+ :values_source_config {:card_id card1-id}}]}]+ (testing "Complain about card not available for exporting"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (extract-aborts! {:targets [["Collection" coll1-id]]+ :no-settings true+ :no-data-model true})+ (is (some #(re-find #"not included in the export" %)+ (into #{}+ (map :message)+ (messages))))))+ (testing "Complain about card depending on an outside card: "+ (testing "when its :source-table"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (extract-aborts! {:targets [["Collection" coll2-id]]+ :no-settings true+ :no-data-model true})+ (is (some #(re-find #"not included in the export" %)+ (into #{}+ (map :message)+ (messages))))))+ (testing "when it's :parameters"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ ;; the parameter-referencing "Card with parameters" lives in coll3, so target coll3 to exercise it+ (extract-aborts! {:targets [["Collection" coll3-id]]+ :no-settings true+ :no-data-model true})+ (is (some #(re-find #"not included in the export" %)+ (into #{}+ (map :message)+ (messages)))))))+ (testing "When exporting all collections"+ (testing "Complain about dependents in personal collections"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (extract-aborts! {:no-settings true+ :no-data-model true})+ (is (some #(re-find #"not included in the export" %)+ (into #{}+ (map :message)+ (messages)))))))+ (testing "exporting a clean collection works even when other collections have escape issues"+ (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]+ :no-settings true+ :no-data-model true+ :no-transforms true}))]+ (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))+ (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))+ (testing "data-model-only export is not blocked by dependency validation"+ (let [extracted (into [] (extract/extract {:no-collections true+ :no-settings true+ :no-transforms true}))]+ (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))+ "Databases should be exported even when cards reference personal collections"))))))++(deftest escape-continue-on-error-test+ (testing "continue-on-error lets the export proceed past dependency validation instead of aborting (#74622)"+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}+ :model/Collection {coll-id :id+ coll-eid :entity_id} {:name "Target Collection"}+ :model/Card {clean-eid :entity_id} {:name "Clean Card"+ :collection_id coll-id+ :database_id db-id}+ :model/Dashboard {dash-id :id+ dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}+ ;; this card lives outside the target collection, so it "escapes"+ :model/Card {escaped-eid :entity_id+ escaped-id :id} {:name "Escaped Card" :database_id db-id}+ :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]+ (testing "without the flag, dependency validation aborts the whole export with an error (#75176)"+ (extract-aborts! opts))+ (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]+ (is (= #{coll-eid} (ids-by-model "Collection" extracted)))+ (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))+ (is (contains? (ids-by-model "Card" extracted) clean-eid)+ "the clean card inside the target collection is exported")+ (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))+ "the escaped card outside the target collection is not exported")+ (is (some #(re-find #"not included in the export" %)+ (map :message (messages)))+ "the unsatisfied reference is still logged as a warning")))))))))++(deftest dependency-validation-missing-data-ref-test+ (testing "Export aborts when a card references a data-model row that no longer exists in the appdb (GHY-4010)"+ ;; The card still serializes a portable reference by looking the row up; with the row gone that reference would be+ ;; malformed, so the export must fail fast. Covered across the reference kinds the old lib-walker-only derivation+ ;; missed: a Field or Segment in the query, and a Field reachable only through visualization_settings (a+ ;; JSON-encoded ref the query walkers don't see). (continue-on-error's abort suppression is covered by+ ;; escape-continue-on-error-test.)+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}+ :model/Table {table-id :id} {:db_id db-id :name "T"}+ :model/Collection {coll-id :id} {:name "Target Collection"}]+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true :no-transforms true}]+ (doseq [{:keys [label query-filter viz-settings delete! msg-re]}+ [{:label "a Field referenced in the query"+ :query-filter (fn [field-id _seg-id] [:> [:field field-id nil] 1])+ :delete! (fn [field-id _seg-id] (t2/delete! :model/Field field-id))+ :msg-re #"Field .* is missing from the source database"}+ {:label "a Segment referenced in the query"+ :query-filter (fn [_field-id seg-id] [:segment seg-id])+ :delete! (fn [_field-id seg-id] (t2/delete! :model/Segment seg-id))+ :msg-re #"Segment .* is missing from the source database"}+ {:label "a Field referenced only in visualization_settings"+ :viz-settings (fn [field-id] {:column_settings {(str "[\"ref\",[\"field\"," field-id ",null]]")+ {:column_title "X"}}})+ :delete! (fn [field-id _seg-id] (t2/delete! :model/Field field-id))+ :msg-re #"Field .* is missing from the source database"}]]+ (testing label+ (ts/with-temp-dpc [:model/Field {field-id :id} {:table_id table-id :name "F" :base_type :type/Integer}+ :model/Segment {segment-id :id} {:table_id table-id :name "Seg" :definition {}}+ :model/Card _ {:name "Card"+ :collection_id coll-id+ :database_id db-id+ :table_id table-id+ :query_type :query+ :dataset_query (cond-> {:database db-id+ :type :query+ :query {:source-table table-id}}+ query-filter+ (assoc-in [:query :filter]+ (query-filter field-id segment-id)))+ :visualization_settings (if viz-settings+ (viz-settings field-id)+ {})}]+ (testing "with the reference present, the export succeeds"+ (is (seq (into [] (extract/extract opts)))))+ (delete! field-id segment-id)+ (testing "after the reference is deleted, the export aborts and names it"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (extract-aborts! opts)+ (is (some #(re-find msg-re %) (map :message (messages)))+ "the warning reports the unsatisfied data-model reference")))))))))))++(deftest serialization-dependencies-content-models-test+ (testing "serialization-dependencies derives export deps from the raw entity for the newly-covered content models (GHY-4010)"+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}+ :model/Table {table-id :id} {:db_id db-id :name "T"}+ :model/Field {field-id :id} {:table_id table-id :name "F" :base_type :type/Integer}+ :model/Collection {parent-id :id} {:name "Parent"}+ :model/Collection {child-id :id} {:name "Child" :location (str "/" parent-id "/")}+ :model/Collection {snip-coll-id :id} {:name "Snips" :namespace "snippets"}+ :model/Card {model-card-id :id} {:name "Model" :type :model :database_id db-id}+ :model/Card {embed-card-id :id} {:name "Embedded" :database_id db-id}]+ (let [deps (fn [model id]+ (serdes/serialization-dependencies model (t2/select-one (keyword "model" model) :id id)))]+ (testing "Collection derives its parent from the raw :location path"+ (is (= #{[{:model "Collection" :id parent-id}]} (deps "Collection" child-id)))+ (is (empty? (deps "Collection" parent-id)) "a root collection has no parent dependency"))+ (testing "Timeline references its containing collection"+ (let [tl-id (t2/insert-returning-pk! :model/Timeline {:name "TL" :icon "star" :collection_id child-id :creator_id (mt/user->id :rasta)})]+ (is (= #{[{:model "Collection" :id child-id}]} (deps "Timeline" tl-id)))))+ (testing "NativeQuerySnippet references its containing collection"+ (let [snip-id (t2/insert-returning-pk! :model/NativeQuerySnippet+ {:name "snip" :content "1=1" :collection_id snip-coll-id+ :creator_id (mt/user->id :rasta)})]+ (is (= #{[{:model "Collection" :id snip-coll-id}]} (deps "NativeQuerySnippet" snip-id)))))+ (testing "Action (query) references its model Card, Database, and the tables/fields in its query"+ (let [action-id (t2/insert-returning-pk! :model/Action {:name "A" :type :query :model_id model-card-id})]+ (t2/insert! :model/QueryAction {:action_id action-id+ :database_id db-id+ :dataset_query {:database db-id+ :type :query+ :query {:source-table table-id+ :filter [:> [:field field-id nil] 1]}}})+ (is (= #{[{:model "Card" :id model-card-id}]+ [{:model "Database" :id db-id}]+ [{:model "Table" :id table-id}]+ [{:model "Field" :id field-id}]}+ (deps "Action" action-id)))))+ (testing "Document references its embedded cards and its containing collection"+ (let [doc-id (t2/insert-returning-pk! :model/Document+ {:name "D"+ :collection_id child-id+ :creator_id (mt/user->id :rasta)+ :content_type prose-mirror/prose-mirror-content-type+ :document {:type "doc"+ :content [{:type "cardEmbed"+ :attrs {:id embed-card-id}}]}})]+ (is (= #{[{:model "Collection" :id child-id}]+ [{:model "Card" :id embed-card-id}]}+ (deps "Document" doc-id))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 71d5ca04052..7099fcbda06 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -526,7 +526,7 @@[{:id model-eid :model "Card"}][{:id card-eid :model "Card"}][{:id "Linked database" :model "Database"}]}- (set (serdes/dependencies extracted-dashboard))))+ (set (serdes/deserialization-dependencies extracted-dashboard))))(storage/store! (seq extraction) (storage.files/file-writer dump-dir))))(testing "ingest and load";; ingestdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 3f5f6c8daab..c507a46b1e0 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -330,7 +330,7 @@(testing "cards depend on their Collection and the query's Database (not the Tables/Fields it references)"(is (= #{[{:model "Database" :id "My Database"}][{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser))))))+ (set (serdes/deserialization-dependencies ser))))))(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c2-id))](is (=? {:serdes/meta [{:model "Card" :id c2-eid :label "second_question"}]:creator_id "[email protected]"@@ -349,7 +349,7 @@(is (= #{[{:model "Database" :id "My Database"}][{:model "Collection" :id coll-eid}][{:model "Card" :id c1-eid}]}- (set (serdes/dependencies ser))))))+ (set (serdes/deserialization-dependencies ser))))))(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c3-id))](is (=? {:serdes/meta [{:model "Card" :id c3-eid :label "third_question"}]:creator_id "[email protected]"@@ -382,7 +382,7 @@(testing "cards depend on their Database (kept, query empty) and Collection (not the Fields in visualization_settings)"(is (= #{[{:model "Database" :id "My Database"}][{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "Cards can be based on other cards"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c5-id))](is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]@@ -400,7 +400,7 @@(is (= #{[{:model "Database" :id "My Database"}][{:model "Collection" :id coll-eid}][{:model "Card" :id c4-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "Dashboards include their Dashcards"(let [ser (ts/extract-one "Dashboard" other-dash-id)](is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]@@ -433,7 +433,7 @@[{:model "Action" :id action-eid}][{:model "Database" :id "My Database"}][{:model "Collection" :id dave-coll-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "Dashboards with parameters where the source is a card"(let [ser (ts/extract-one "Dashboard" param-dash-id)](is (=? {:parameters@@ -450,7 +450,7 @@[{:model "Card" :id c1-eid}];; the parameter's value_field references a Field, but only its Database is a dependency[{:model "Database", :id "My Database"}]}- (set (serdes/dependencies ser))))))+ (set (serdes/deserialization-dependencies ser))))))(testing "Cards with parameters where the source is a card"(let [ser (ts/extract-one "Dashboard" param-dash-id)](is (=? {:parameters@@ -467,7 +467,7 @@[{:model "Card" :id c1-eid}];; the parameter's value_field references a Field, but only its Database is a dependency[{:model "Database", :id "My Database"}]}- (set (serdes/dependencies ser))))))+ (set (serdes/deserialization-dependencies ser))))))(testing "collection filtering based on :user option"(testing "only unowned collections are returned with no user"(is (= ["Some Collection"]@@ -526,7 +526,7 @@[{:model "Card" :id c2-eid}][{:model "Card" :id c3-eid}][{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))(deftest dimensions-test(mt/with-empty-h2-app-db!@@ -575,7 +575,7 @@(->> ser :dimensions (map :entity_id)))))(testing "depend only on the Database; the Table is synthesized on import if missing"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "foreign key dimensions are inlined into their Fields"(let [ser (ts/extract-one "Field" fk-id)](is (malli= [:map@@ -597,7 +597,7 @@(:dimensions ser))))(testing "depend only on the Database; the Table, FK target and human-readable Fields are synthesized on import if missing"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))(deftest native-query-snippets-test(mt/with-empty-h2-app-db!@@ -644,7 +644,7 @@(is (not (contains? ser :id)))(testing "and depend on the Collection"(is (= #{[{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser)))))+ (set (serdes/deserialization-dependencies ser)))))(testing "and will bring collection to extraction"(is (= {["Collection" coll-id] {"NativeQuerySnippet" s1-id}}(serdes/required "NativeQuerySnippet" s1-id))))))@@ -660,7 +660,7 @@ser))(is (not (contains? ser :id)))(testing "and has no deps"- (is (empty? (serdes/dependencies ser))))))+ (is (empty? (serdes/deserialization-dependencies ser))))))(testing "Snippet collection is exported when snippet is exported as a card dep (#51901)"(is (= {["Collection" coll2-id] nil["Card" card-id] {"Collection" coll2-id}@@ -764,7 +764,7 @@(is (not (contains? ser :id)))(testing "depend on the Collection"(is (= #{[{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "with events"(let [ser (ts/extract-one "Timeline" line-id)](is (=? {:serdes/meta [{:model "Timeline" :id line-eid :label "populated_timeline"}]@@ -779,7 +779,7 @@(is (not (contains? (-> ser :events first) :id)))(testing "depend on the Collection"(is (= #{[{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser)))))))))))+ (set (serdes/deserialization-dependencies ser)))))))))))(deftest segments-test(mt/with-empty-h2-app-db!@@ -812,7 +812,7 @@(is (not (contains? ser :id)))(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))(defn- mbql5-measure-definition"Create an MBQL5 measure definition with a sum aggregation."@@ -853,7 +853,7 @@(is (not (contains? ser :id)))(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser))))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))))(deftest measure-referencing-measure-test(mt/with-empty-h2-app-db!@@ -894,7 +894,7 @@:aggregation [[:* {} [:measure {} m1-eid] 2]]}]}}ser))(testing "depends on the referenced Measure"- (is (contains? (set (serdes/dependencies ser))+ (is (contains? (set (serdes/deserialization-dependencies ser))[{:model "Measure" :id m1-eid}]))))))))))))(deftest measure-referencing-segment-test@@ -936,7 +936,7 @@:aggregation [[:count-where {} [:segment {} seg-eid]]]}]}}ser))(testing "depends on the referenced Segment"- (is (contains? (set (serdes/dependencies ser))+ (is (contains? (set (serdes/deserialization-dependencies ser))[{:model "Segment" :id seg-eid}]))))))))))(deftest table-publishing-serdes-test@@ -959,7 +959,7 @@(testing "collection_id is transformed to entity_id"(is (= coll-eid (:collection_id ser))))(testing "depends on the collection"- (is (contains? (set (serdes/dependencies ser))+ (is (contains? (set (serdes/deserialization-dependencies ser))[{:model "Collection" :id coll-eid}])))))(testing "unpublished table without collection_id"(let [ser (ts/extract-one "Table" unpub-table-id)]@@ -969,7 +969,7 @@(is (nil? (:collection_id ser))))(testing "does not depend on any collection"(is (not (some #(= "Collection" (:model (first %)))- (serdes/dependencies ser)))))))+ (serdes/deserialization-dependencies ser)))))))(testing "regular table without publishing fields set"(let [ser (ts/extract-one "Table" table-id)](testing "is_published is omitted (default false)"@@ -1011,7 +1011,7 @@(is (not (contains? ser :id)))(testing "depends on the Model"(is (= #{[{:model "Card" :id card-eid-1}]}- (set (serdes/dependencies ser)))))))))))))+ (set (serdes/deserialization-dependencies ser)))))))))))))(deftest query-action-test(mt/with-empty-h2-app-db!@@ -1054,7 +1054,7 @@(testing "depends on the Model and Database"(is (= #{[{:model "Database" :id "My Database"}][{:model "Card" :id card-eid-1}]}- (set (serdes/dependencies ser)))))))))))))+ (set (serdes/deserialization-dependencies ser)))))))))))))(deftest field-values-test(mt/with-empty-h2-app-db!@@ -1093,7 +1093,7 @@":field_id is dropped; its implied by the path")(testing "depend only on the Database; the parent Field is synthesized on import if missing"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "extract-metabase behavior"(testing "without :include-field-values"(is (= #{}@@ -1125,7 +1125,7 @@":field_id is dropped; its implied by the path")(testing "depend only on the Database; the parent Field is synthesized on import if missing"(is (= #{[{:model "Database" :id "My Database"}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "extract-metabase behavior"(let [models (->> {} (extract/extract) (map (comp :model last :serdes/meta)))](is (= 1@@ -1183,7 +1183,7 @@(is (= #{[{:model "Database" :id "My Database"}][{:model "Collection" :id coll-eid-2}][{:model "Card" :id card-eid-1}]}- (set (serdes/dependencies ser))))))+ (set (serdes/deserialization-dependencies ser))))))(testing "Nullable transformations are omitted"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id card-id-2))](is (not (contains? ser :made_public_by_id))))))))@@ -1465,8 +1465,8 @@(let [msgs (into #{}(map :message)(messages))]- (is (some #(str/starts-with? % "Failed to export Dashboard") msgs))- (is (some #(str/starts-with? % "Failed to export Cards") msgs))))))))))+ (is (some #(re-find #"not included in the export" %) msgs)+ "the export warns that a referenced personal-collection card is not included")))))))))(deftest click-behavior-references-to-deleted-cards(mt/with-empty-h2-app-db!@@ -1570,124 +1570,6 @@{:model "Field", :id "Nested Field"}](:serdes/meta (ts/extract-one "Field" nested-id))))))))-(deftest escape-report-test- (mt/with-empty-h2-app-db!- (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}- :model/Collection {coll2-id :id} {:name "Other Collection"}- :model/Collection {coll3-id :id} {:name "Third Collection"}- :model/Collection {clean-coll-id :id- clean-coll-eid :entity_id} {:name "Clean Collection"}- :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}- ;; non-H2 engine so the database survives serdes extract filtering- :model/Database {db-id :id} {:engine :postgres}- :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}- :model/Card {clean-card-eid :entity_id} {:name "Clean Card"- :collection_id clean-coll-id- :database_id db-id}- :model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}- :model/Card _ {:name "Dependent Card"- :collection_id coll2-id- :dataset_query {:type :query- :database db-id- :query {:source-table (str "card__" card1-id)}}}- :model/User user {:email "[email protected]"}- :model/Collection pcoll {:name "Personal Collection"- :personal_owner_id (:id user)}- :model/Card pcard {:name "Personal Card"- :collection_id (:id pcoll)}- :model/Card _ {:name "External Card"- :dataset_query {:database db-id- :type :query- :query {:source-table (str "card__" (:id pcard))}}}- :model/Card _ {:name "Card with parameters"- :collection_id coll3-id- :parameters [{:id "abc"- :type "category"- :values_source_type "card"- :values_source_config {:card_id card1-id}}]}]- (testing "Complain about card not available for exporting"- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract-aborts! {:targets [["Collection" coll1-id]]- :no-settings true- :no-data-model true})- (is (some #(str/starts-with? % "Failed to export Dashboard")- (into #{}- (map :message)- (messages))))))- (testing "Complain about card depending on an outside card: "- (testing "when its :source-table"- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract-aborts! {:targets [["Collection" coll2-id]]- :no-settings true- :no-data-model true})- (is (some #(str/starts-with? % "Failed to export Cards")- (into #{}- (map :message)- (messages))))))- (testing "when it's :parameters"- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract-aborts! {:targets [["Collection" coll2-id]]- :no-settings true- :no-data-model true})- (is (some #(str/starts-with? % "Failed to export Cards")- (into #{}- (map :message)- (messages)))))))- (testing "When exporting all collections"- (testing "Complain about dependents in personal collections"- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract-aborts! {:no-settings true- :no-data-model true})- (is (some #(str/starts-with? % "Failed to export Cards")- (into #{}- (map :message)- (messages)))))))- (testing "exporting a clean collection works even when other collections have escape issues"- (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]- :no-settings true- :no-data-model true- :no-transforms true}))]- (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))- (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))- (testing "data-model-only export is not blocked by escape analysis"- (let [extracted (into [] (extract/extract {:no-collections true- :no-settings true- :no-transforms true}))]- (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))- "Databases should be exported even when cards reference personal collections"))))))--(deftest escape-continue-on-error-test- (testing "continue-on-error lets the export proceed past escape analysis instead of aborting (#74622)"- (mt/with-empty-h2-app-db!- (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering- :model/Database {db-id :id} {:engine :postgres}- :model/Collection {coll-id :id- coll-eid :entity_id} {:name "Target Collection"}- :model/Card {clean-eid :entity_id} {:name "Clean Card"- :collection_id coll-id- :database_id db-id}- :model/Dashboard {dash-id :id- dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}- ;; this card lives outside the target collection, so it "escapes"- :model/Card {escaped-eid :entity_id- escaped-id :id} {:name "Escaped Card" :database_id db-id}- :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]- (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]- (testing "without the flag, escape analysis aborts the whole export with an error (#75176)"- (extract-aborts! opts))- (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"- (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]- (is (= #{coll-eid} (ids-by-model "Collection" extracted)))- (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))- (is (contains? (ids-by-model "Card" extracted) clean-eid)- "the clean card inside the target collection is exported")- (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))- "the escaped card outside the target collection is not exported")- (is (some #(str/starts-with? % "Failed to export Dashboard")- (map :message (messages)))- "the escape report is still logged as a warning")))))))))-(deftest recursive-colls-test(mt/with-empty-h2-app-db!(mt/with-temp [:model/Collection {parent-id :id@@ -1951,7 +1833,7 @@(testing "metabot depends on its model entities and collection"(is (= #{[{:model "Card" :id model-eid}][{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser)))))+ (set (serdes/deserialization-dependencies ser)))))(testing "metabot storage-path uses top-level metabots directory"(is (= [{:label "metabots"} {:label "Test Metabot" :key metabot-eid}](serdes/storage-path ser {})))))))))@@ -1996,7 +1878,7 @@(testing "metabot depends on its prompts' cards and its collection"(is (= #{[{:model "Card" :id card-eid}][{:model "Collection" :id coll-eid}]}- (set (serdes/dependencies ser))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))(deftest document-test(mt/with-empty-h2-app-db!@@ -2058,7 +1940,7 @@[{:model "Dashboard" :id (:entity_id dashboard)}][{:model "Card" :id (:entity_id linked-card)}](serdes/generate-path "Table" table)}- (set (serdes/dependencies ser))))))))))+ (set (serdes/deserialization-dependencies ser))))))))))(deftest visualizer-dashboard-card-settings-test(testing "visualizer settings transform entity IDs <-> card IDs"@@ -2136,7 +2018,7 @@:created_at string?}ser))(is (not (contains? ser :id)))- (is (empty? (serdes/dependencies ser)))))+ (is (empty? (serdes/deserialization-dependencies ser)))))(testing "custom tags extract correctly"(let [ser (serdes/extract-one "TransformTag" {} (t2/hydrate (t2/select-one :model/TransformTag :id custom-tag-id) :tags))](is (=? {:serdes/meta [{:model "TransformTag"@@ -2146,7 +2028,7 @@ser))(is (not (contains? ser :built_in_type)))(is (not (contains? ser :id)))- (is (empty? (serdes/dependencies ser)))))+ (is (empty? (serdes/deserialization-dependencies ser)))))(testing "all transform tags are extracted"(is (= #{hourly-tag-eid daily-tag-eid custom-tag-eid}(ids-by-model "TransformTag" (extract/extract {}))))))))))@@ -2255,7 +2137,7 @@(is (= [hourly-tag-eid custom-tag-eid daily-tag-eid] tag-ids))(is (= [0 1 2] positions))))(testing "dependencies include collection, source database, and tags (the source Table itself is not a dependency)"- (let [deps (set (serdes/dependencies ser))]+ (let [deps (set (serdes/deserialization-dependencies ser))](is (contains? deps [{:model "Collection" :id coll-eid}]))(is (contains? deps [{:model "Database" :id "My Database"}]))(is (not (contains? deps [{:model "Database" :id "My Database"}@@ -2317,7 +2199,7 @@(is (= "SELECT 1" (get-in ser [:source :query :stages 0 :native]))))(testing "Transform/dependencies does not emit a Database dep when source_database_id is nil"(is (not-any? #(some (fn [{:keys [model]}] (= "Database" model)) %)- (serdes/dependencies (assoc reloaded :tags [])))))))))))+ (serdes/deserialization-dependencies (assoc reloaded :tags [])))))))))))(deftest table-with-transform-id-dependency-test(testing "Table created by a Transform declares the Transform as a serdes dependency (GDGT-2444)"@@ -2349,7 +2231,7 @@(testing "transform_id is transformed to entity_id"(is (= transform-eid (:transform_id ser))))(testing "depends on the transform"- (is (contains? (set (serdes/dependencies ser))+ (is (contains? (set (serdes/deserialization-dependencies ser))[{:model "Transform" :id transform-eid}])))))))))(deftest transform-job-extraction-test@@ -2420,7 +2302,7 @@(is (= 0 (-> ser :job_tags first :position))))(testing "dependencies include referenced tags"(is (= #{[{:model "TransformTag" :id hourly-tag-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "custom job extracts correctly"(let [ser (serdes/extract-one "TransformJob" {} (t2/hydrate (t2/select-one :model/TransformJob :id custom-job-id) :job_tags))](is (=? {:serdes/meta [{:model "TransformJob"@@ -2441,7 +2323,7 @@(testing "dependencies include all referenced tags"(is (= #{[{:model "TransformTag" :id custom-tag-eid}][{:model "TransformTag" :id daily-tag-eid}]}- (set (serdes/dependencies ser)))))))+ (set (serdes/deserialization-dependencies ser)))))))(testing "all transform jobs are extracted"(is (= #{hourly-job-eid custom-job-eid}(ids-by-model "TransformJob" (extract/extract {}))))))))))@@ -2543,7 +2425,7 @@ser))(is (not (contains? ser :id)))(testing "has no dependencies"- (is (empty? (serdes/dependencies ser)))))))))+ (is (empty? (serdes/deserialization-dependencies ser)))))))))(deftest custom-viz-plugin-test(mt/with-empty-h2-app-db!@@ -2570,7 +2452,7 @@(is (not (contains? ser :id)))(is (not (contains? ser :status)))(testing "has no dependencies"- (is (empty? (serdes/dependencies ser)))))))))+ (is (empty? (serdes/deserialization-dependencies ser)))))))))(deftest ^:parallel export-parameters-sorts-by-id-test(let [params [{:id "zebra" :name "Z param" :type :category}@@ -2770,7 +2652,7 @@:dataset_query (mbql5-query db-id table-id):display :table}](let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card card-id))- deps (serdes/dependencies ser)]+ deps (serdes/deserialization-dependencies ser)];; Database dep comes from mbql-deps on the query's :database key(is (contains? (set deps) [{:model "Database" :id "Test DB"}])"Database dependency should come from the query")@@ -2841,7 +2723,7 @@ser))(is (not (contains? ser :id)))(testing "embedding themes have no dependencies"- (is (empty? (serdes/dependencies ser))))))+ (is (empty? (serdes/deserialization-dependencies ser))))))(testing "all embedding themes are extracted"(is (= #{light-eid dark-eid}(ids-by-model "EmbeddingTheme" (extract/extract {}))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex aa0f7a93c9d..d2d5b122048 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -609,7 +609,7 @@derived-measure))(is (= #{[{:id "my-db", :model "Database"}][{:id (:entity_id @msr1s), :model "Measure"}]}- (serdes/mbql-deps (:definition derived-measure))))))+ (serdes/mbql-deps false (:definition derived-measure))))))(testing "deserializing adjusts the measure IDs properly"(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.cljindex a5000959b2f..c79140844ad 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/models_test.clj@@ -138,7 +138,7 @@(def ^:private auto-synthesized-fk-targets"FK target models whose importers auto-create a stub entity if the target is missing, so- they don't have to be declared in `serdes/dependencies`. Everything else MUST be declared+ they don't have to be declared in `serdes/deserialization-dependencies`. Everything else MUST be declared— otherwise `*import-fk*` throws \"Could not find foreign key target\" and aborts thewhole import (see GDGT-2444 for an example)."#{:model/User :model/Table :model/Field})@@ -174,7 +174,7 @@(deftest ^:parallel serialization-direct-fk-dependencies-completeness-test(testing (str "Every direct FK declared via `(serdes/fk ...)` in a loadable (non-inlined) "- "model's `make-spec` is surfaced in that model's `serdes/dependencies` — "+ "model's `make-spec` is surfaced in that model's `serdes/deserialization-dependencies` — ""otherwise `*import-fk*` will throw \"Could not find foreign key target\" at ""import time (see GDGT-2444).")(let [inlined? (set serdes.models/inlined-models)]@@ -194,13 +194,13 @@(let [entity (into {:serdes/meta [{:model m :id "self"}]}(for [[field _] fks][field (str "fake-eid-" (name field))]))- deps (set (serdes/dependencies entity))+ deps (set (serdes/deserialization-dependencies entity))dep-models (set (keep (comp :model peek) deps))](doseq [[field target] fks]- (testing (format "FK `%s` -> `%s` must appear in `(serdes/dependencies)`"+ (testing (format "FK `%s` -> `%s` must appear in `(serdes/deserialization-dependencies)`"field target)(is (contains? dep-models (name target))- (format (str "`(serdes/dependencies %s)` did not include a `%s` entry "+ (format (str "`(serdes/deserialization-dependencies %s)` did not include a `%s` entry ""when `%s` was populated. Add the FK to the model's ""`dependencies` method (see Table/Transform for an example).")m (name target) field))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.cljindex 1a47657caab..c5a49ce43b0 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/osi_ai_context_test.clj@@ -60,7 +60,7 @@extracted))(is (not (contains? extracted :entity_id))))(testing "depends on the referenced Card"- (is (= [[{:model "Card" :id card-eid}]] (serdes/dependencies extracted))))+ (is (= [[{:model "Card" :id card-eid}]] (serdes/deserialization-dependencies extracted))))(testing "importing resolves the path back to the local card row"(t2/delete! :model/OsiAiContext :entity_type "card" :entity_local_id card-id)(serdes.load/load-metabase! (ingestion-in-memory [extracted]))@@ -86,7 +86,7 @@(is (=? {:serdes/meta [{:model serdes-model :id eid} {:model "OsiAiContext" :id "ai_context"}]}extracted)))(testing "depends on the referenced entity"- (is (= [[{:model serdes-model :id eid}]] (serdes/dependencies extracted))))+ (is (= [[{:model serdes-model :id eid}]] (serdes/deserialization-dependencies extracted))))(testing "importing resolves the path back to the local id"(t2/delete! :model/OsiAiContext :entity_type entity-type :entity_local_id id)(serdes.load/load-metabase! (ingestion-in-memory [extracted]))@@ -102,7 +102,7 @@(is (= {:model "OsiAiContext" :id "ai_context"} (self-segment extracted)))(is (= "Table" (:model (last (pop (vec (serdes/path extracted))))))))(testing "depends on the Table"- (is (= [(vec (pop (vec (serdes/path extracted))))] (serdes/dependencies extracted))))+ (is (= [(vec (pop (vec (serdes/path extracted))))] (serdes/deserialization-dependencies extracted))))(testing "importing resolves the path back to the local id"(t2/delete! :model/OsiAiContext :entity_type "table" :entity_local_id table-id)(serdes.load/load-metabase! (ingestion-in-memory [extracted]))diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 0786e5bfc89..bfc1f0a87b6 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -457,7 +457,7 @@:import serdes/import-visualization-settings}}:defaults {:archived false}})-(defmethod serdes/dependencies "Action" [action]+(defmethod serdes/deserialization-dependencies "Action" [action](set(concat;; other stuff is implicitly referenced through a Card@@ -467,7 +467,19 @@(let [{:keys [database_id dataset_query]} (first (:query action))](concat[[{:model "Database" :id database_id}]]- (serdes/mbql-deps dataset_query)))))))+ (serdes/mbql-deps false dataset_query)))))))++(defmethod serdes/serialization-dependencies "Action" [_model-name {:keys [id model_id type]}]+ ;; Serialization runs on the raw entity, whose query lives in the `query_action` child table (`:type` is a keyword+ ;; here, not a string), so the query is fetched rather than read from a nested `:query` key.+ (set+ (concat+ (when model_id [[{:model "Card" :id model_id}]])+ (when (= type :query)+ (when-let [{:keys [database_id dataset_query]} (t2/select-one :model/QueryAction :action_id id)]+ (concat+ (when database_id [[{:model "Database" :id database_id}]])+ (serdes/mbql-deps true dataset_query)))))))(defmethod serdes/storage-path "Action" [action _ctx][{:label "actions"} {:label (:name action) :key (:entity_id action)}])diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex e8e2c90d7b2..c466beafbb8 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -2107,11 +2107,18 @@;; stable filename de-dup suffixes across exports, see GHY-3754:order-by serdes/stable-storage-order}))))-(defmethod serdes/dependencies "Collection"+(defmethod serdes/deserialization-dependencies "Collection"[{:keys [parent_id]}](when parent_id#{[{:model "Collection" :id parent_id}]}))+(defmethod serdes/serialization-dependencies "Collection"+ ;; The raw entity carries its ancestry in `:location` (the ingested form is normalized to `:parent_id`), so the+ ;; parent Collection is derived from the path. A selective export may legitimately omit the parent.+ [_model-name {:keys [location]}]+ (when-let [parent-id (location-path->parent-id location)]+ #{[{:model "Collection" :id parent-id}]}))+(defmethod serdes/generate-path "Collection" [_ coll](serdes/maybe-labeled "Collection" coll :slug))diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex cbef429b98c..d9f59cf1c1f 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -445,21 +445,27 @@:show_in_getting_started false}})(defn- serdes-deps-dashcard- [{:keys [action_id card_id parameter_mappings visualization_settings series]}]+ [allow-int-ids? {:keys [action_id card_id parameter_mappings visualization_settings series]}](set(concat- (mapcat serdes/mbql-deps parameter_mappings)- (serdes/visualization-settings-deps visualization_settings)+ (mapcat #(serdes/mbql-deps allow-int-ids? %) parameter_mappings)+ (serdes/visualization-settings-deps allow-int-ids? visualization_settings)(when card_id #{[{:model "Card" :id card_id}]})(when action_id #{[{:model "Action" :id action_id}]})(for [s series] [{:model "Card" :id (:card_id s)}]))))-(defmethod serdes/dependencies "Dashboard"- [{:keys [collection_id dashcards parameters]}]- (->> (map serdes-deps-dashcard dashcards)+(defn- dashboard-deps+ "The serdes dependencies of a Dashboard as `:serdes/meta` paths. Expects `:dashcards` (with `:series`) inlined — the+ ingested shape, which the raw path hydrates first. `allow-int-ids?` selects raw-appdb vs serialized ref semantics. Shared by+ [[serdes/deserialization-dependencies]] (allow-int-ids? false) and [[serdes/serialization-dependencies]] (allow-int-ids? true)."+ [allow-int-ids? {:keys [collection_id dashcards parameters]}]+ (->> (map #(serdes-deps-dashcard allow-int-ids? %) dashcards)(reduce set/union #{})(set/union (when collection_id #{[{:model "Collection" :id collection_id}]}))- (set/union (serdes/parameters-deps parameters))))+ (set/union (serdes/parameters-deps allow-int-ids? parameters))))++(defmethod serdes/deserialization-dependencies "Dashboard" [dashboard]+ (dashboard-deps false dashboard))(defmethod serdes/descendants "Dashboard" [_model-name id _opts](let [dashcards (t2/select [:model/DashboardCard :id :card_id :action_id :parameter_mappings :visualization_settings]@@ -493,6 +499,13 @@(into {} (for [card-id (some->> dashboard :parameters (keep (comp :card_id :values_source_config)))]{["Card" card-id] {"Dashboard" dash-id}})))))+(defmethod serdes/serialization-dependencies "Dashboard" [_model-name dashboard]+ ;; a raw Dashboard has its dashcards/series in separate tables; hydrate them into the inlined shape dashboard-deps+ ;; expects, then reuse the same walk as deserialization.+ (dashboard-deps true (-> dashboard+ (t2/hydrate :dashcards)+ (update :dashcards #(t2/hydrate % :series)))))+;;;; ------------------------------------------------- Search ----------------------------------------------------------(search/define-spec "dashboard"diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.cljindex ce19a582abd..25339e87376 100644--- a/src/metabase/documents/models/document.clj+++ b/src/metabase/documents/models/document.clj@@ -245,12 +245,30 @@:elsenil))))))-(defmethod serdes/dependencies "Document"+(defmethod serdes/deserialization-dependencies "Document"[{:keys [collection_id] :as document}](set (concat(document-deps document)(when collection_id #{[{:model "Collection" :id collection_id}]}))))+(defmethod serdes/serialization-dependencies "Document"+ ;; Embedded cards and smart-links become content references (each must be part of the export); the containing+ ;; Collection is included too, though a selective export may legitimately omit it.+ [_model-name {:keys [collection_id content_type] :as document}]+ (set+ (concat+ (when collection_id [[{:model "Collection" :id collection_id}]])+ (when (= content_type prose-mirror/prose-mirror-content-type)+ (concat+ (for [embedded-card-id (prose-mirror/card-ids document)]+ [{:model "Card" :id embedded-card-id}])+ (for [{model :model link-id :entityId}+ (prose-mirror/collect-ast document+ #(when (= prose-mirror/smart-link-type (:type %))+ (:attrs %)))+ :when (contains? model->serdes-model model)]+ [{:model (model->serdes-model model) :id link-id}]))))))+(defmethod serdes/descendants "Document"[_model-name id _opts](when-let [document (t2/select-one :model/Document :id id)]diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex bca7ecb20e0..0ad66be8a2d 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -190,8 +190,8 @@[_measure][:name (serdes/hydrated-hash :table) :created_at])-(defmethod serdes/dependencies "Measure" [{:keys [definition]}]- (serdes/mbql-deps definition))+(defmethod serdes/deserialization-dependencies "Measure" [{:keys [definition]}]+ (serdes/mbql-deps false definition))(defmethod serdes/storage-path "Measure" [measure _ctx](let [table-path (-> measure :definition serdes/serialized-query-source-table)]diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.cljindex 49369214244..5b8b6740fee 100644--- a/src/metabase/metabot/models/metabot.clj+++ b/src/metabase/metabot/models/metabot.clj@@ -41,9 +41,9 @@[_table][:name])-(defmethod serdes/dependencies "Metabot"+(defmethod serdes/deserialization-dependencies "Metabot"[{:keys [collection_id prompts]}]- (cond-> (set (mapcat serdes/dependencies prompts))+ (cond-> (set (mapcat serdes/deserialization-dependencies prompts))collection_id (conj [{:model "Collection" :id collection_id}])))(defmethod serdes/generate-path "Metabot" [_ metabot]diff --git a/src/metabase/metabot/models/metabot_prompt.clj b/src/metabase/metabot/models/metabot_prompt.cljindex 2cdc0c12c99..f1f780d775a 100644--- a/src/metabase/metabot/models/metabot_prompt.clj+++ b/src/metabase/metabot/models/metabot_prompt.clj@@ -28,7 +28,7 @@(conj (serdes/generate-path "Metabot" (t2/select-one :model/Metabot (:metabot_id entity)))(serdes/infer-self-path "MetabotPrompt" entity)))-(defmethod serdes/dependencies "MetabotPrompt" [prompt]+(defmethod serdes/deserialization-dependencies "MetabotPrompt" [prompt]#{[{:model "Card" :id (:card_id prompt)}]})(defmethod serdes/make-spec "MetabotPrompt" [_model-name _opts]diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 301a15d8c90..c99b5a8941f 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -24,7 +24,7 @@tests)- `serdes/descendants` - if model should be extracted along with some support (like Collection and all itschildren), you need to specify what to fetch - see `Card` for an example (used during export)- - `serdes/dependencies` - if model references other models, you need to declare which ones, see `Action` for an+ - `serdes/deserialization-dependencies` - if model references other models, you need to declare which ones, see `Action` for anexample (used during import)- Write tests- basic layout (that all entity fields are mentioned, fks are marked as such) is tested automatically@@ -86,6 +86,10 @@;; there was no science behind picking 100 as a number(def ^:private extract-nested-batch-limit "max amount of entities to fetch nested entities for" 100)+(def query-batch-size+ "Maximum number of ids per `:in` clause, to stay under database parameter limits."+ 1000)+(mr/def ::model-keyword[:andqualified-keyword?@@ -654,6 +658,19 @@(defmethod required :default [_ _]nil)+(defmulti serialization-dependencies+ "The references of a raw appdb `entity` (a Toucan instance, not a serialized map) as a seq of dependency maps+ `{:model \"ModelName\", :id id}`. Its load-time counterpart is [[deserialization-dependencies]], which runs on the+ serialized form and omits tables/fields (import creates them on the fly); this one reports them so the export can+ existence-check them. Consumed by [[metabase-enterprise.serialization.v2.dependency-validation]].++ Dispatched on model-name. Default returns `nil`, so only models with references need to implement this."+ {:arglists '([model-name entity])}+ (fn [model-name _entity] model-name))++(defmethod serialization-dependencies :default [_ _]+ nil)+;;; # Metadata Export(defmulti metadata-query@@ -713,7 +730,7 @@;;; Then for each ingested entity:;;;;;; - `(ingest-one serdes-path opts)` is called to read the value into memory, then-;;; - `(dependencies ingested)` gets a list of other `:serdes/meta` paths need to be loaded first.+;;; - `(deserialization-dependencies ingested)` gets a list of other `:serdes/meta` paths need to be loaded first.;;; - See below on dependencies.;;; - Dependencies are loaded recursively in postorder; that is an entity is loaded after all its deps.;;; - Circular dependencies will make the load process throw.@@ -766,7 +783,7 @@;;; ## Dependencies;;; The files of an export are returned in arbitrary order by [[ingest-list]]. But in order to load any entity,;;; everything it has a foreign key to must be loaded first. This is the purpose of one of the most complicated parts of-;;; serdes: [[dependencies]].+;;; serdes: [[deserialization-dependencies]].;;;;;; This multimethod returns a list (possibly empty) of `:serdes/meta` paths that this entity depends on. A `Card`;;; depends on the `Table`s it queries, the `Collection` it belongs to, and possibly much else.@@ -777,18 +794,19 @@;;; Missing dependencies will cause flaky deserialization failures, since sometimes the FK target will exist already,;;; and sometimes not, depending on the arbitrary order of `ingest-list`.-(defmulti dependencies+(defmulti deserialization-dependencies"Given an entity map as ingested (not a Toucan entity) returns a (possibly empty) list of its dependencies, where eachdependency is represented by its abstract path (its `:serdes/meta` value).- NOTE: This is called during **LOAD**.+ NOTE: This is called during **LOAD**. Its export-time counterpart is [[serialization-dependencies]], which runs on a+ raw entity and additionally reports tables/fields (which import synthesizes on the fly, so they aren't load deps).Keyed on the model name for this entity.Default implementation returns `nil`, so only models that have dependencies need to implement this."{:arglists '([ingested])}ingested-model)-(defmethod dependencies :default [_]+(defmethod deserialization-dependencies :default [_]nil)(defmulti load-update!@@ -1030,7 +1048,7 @@The identifier can be a single entity ID string, a single identity-hash string, or a vector of entity ID and hashstrings. If the ID is compound, then the last ID is the one that corresponds to the model. This allows for the- compound IDs needed for nested entities like `DashboardCard`s to get their [[dependencies]].+ compound IDs needed for nested entities like `DashboardCard`s to get their [[deserialization-dependencies]].Throws if the corresponding entity cannot be found.@@ -1111,7 +1129,7 @@(defn table->path"Given a `table_id` as exported by [[export-table-fk]], turn it into a `[{:model ...}]` path for the Table.- This is useful for writing [[dependencies]] implementations."+ This is useful for writing [[deserialization-dependencies]] implementations."[[db-name schema table-name]](filterv some? [{:model "Database" :id db-name}(when schema {:model "Schema" :id schema})@@ -1198,7 +1216,7 @@(defn field->path"Given a `field_id` as exported by [[export-field-fk]], turn it into a `[{:model ...}]` path for the Field.- This is useful for writing [[dependencies]] implementations."+ This is useful for writing [[deserialization-dependencies]] implementations."[[db-name schema table-name field-name]](filterv some? [{:model "Database" :id db-name}(when schema {:model "Schema" :id schema})@@ -1437,6 +1455,14 @@(declare ^:private mbql-deps-map)+(defn- raw-ref-id?+ "True if `x` is a raw (numeric) reference id — i.e. a positive integer — and `allow-int-ids?` is set, meaning we're computing+ *serialization* (export) deps from a raw entity. The `mbql-*` walkers serve both forms; a numeric is only treated as+ a reference when `allow-int-ids?`, since a serialized query can still carry a stray numeric id (e.g. a query whose deleted+ source database never got name-ified) that must not be mistaken for a live ref at load time."+ [allow-int-ids? x]+ (and allow-int-ids? (pos-int? x)))+(defn- ref->db-dep"Given a portable table or field reference (a vector like `[db-name schema table-name ...]`), return a set withits Database dependency, or nil. Table and Field references are intentionally *not* dependencies — missing ones@@ -1447,49 +1473,68 @@(when-let [db-name (first ref)]#{[{:model "Database" :id db-name}]}))-(defn- mbql-deps-vector [entity]+(def ^:private mbql-ref-tag->model+ "The serdes model that a `:metric`/`:segment`/`:measure` MBQL reference clause depends on."+ {:metric "Card", "metric" "Card"+ :segment "Segment", "segment" "Segment"+ :measure "Measure", "measure" "Measure"})++(defn- mbql-deps-vector [allow-int-ids? entity](match/match-one entity- ;; A `:field`/`:field-id` clause's only dependency is the Database of its referenced field; the Field itself- ;; is not a dependency, and a field clause never contains nested metric/segment/card refs, so we don't descend.+ ;; --- serialized (portable) refs, walked at load time ---+ ;; A serialized `:field` clause's only dependency is the Database of its referenced field; the Field/Table+ ;; themselves are synthesized on import, and a field clause never nests metric/segment/card refs, so we don't+ ;; descend.[#{:field "field"} (_opts :guard map?) (ref :guard vector?)](ref->db-dep ref)[(tag :guard #{:metric "metric" :segment "segment" :measure "measure"})(opts :guard map?)(field :guard portable-id?)]- (into #{[{:model (case tag- (:metric "metric") "Card"- (:segment "segment") "Segment"- (:measure "measure") "Measure")- :id field}]}- (mbql-deps-map opts))-- ;; legacy (MBQL 4) refs+ (into #{[{:model (mbql-ref-tag->model tag) :id field}]}+ (mbql-deps-map allow-int-ids? opts))++ ;; legacy (MBQL 4) serialized refs[#{:field "field" :field-id "field-id"} (ref :guard vector?) _opts](ref->db-dep ref)[(tag :guard #{:metric "metric" :segment "segment" :measure "measure"}) (field :guard portable-id?)]- #{[{:model (case tag- (:metric "metric") "Card"- (:segment "segment") "Segment"- (:measure "measure") "Measure")- :id field}]}+ #{[{:model (mbql-ref-tag->model tag) :id field}]}++ ;; --- raw (numeric) refs, walked at export time before serialization ---+ ;; The referenced Field/Table are real appdb ids, existence-checked so a deleted row can't produce a malformed+ ;; portable reference. `allow-int-ids?` gates these so a stray numeric in serialized data isn't mistaken for a ref.+ [#{:field "field"} (_opts :guard map?) (id :guard (raw-ref-id? allow-int-ids? id))]+ #{[{:model "Field" :id id}]}++ [(tag :guard #{:metric "metric" :segment "segment" :measure "measure"})+ (opts :guard map?)+ (id :guard (raw-ref-id? allow-int-ids? id))]+ (into #{[{:model (mbql-ref-tag->model tag) :id id}]}+ (mbql-deps-map allow-int-ids? opts))++ [#{:field "field" :field-id "field-id"} (id :guard (raw-ref-id? allow-int-ids? id)) _opts]+ #{[{:model "Field" :id id}]}++ [(tag :guard #{:metric "metric" :segment "segment" :measure "measure"}) (id :guard (raw-ref-id? allow-int-ids? id))]+ #{[{:model (mbql-ref-tag->model tag) :id id}]}_ (reduce #(cond- (map? %2) (into %1 (mbql-deps-map %2))- (vector? %2) (into %1 (mbql-deps-vector %2))+ (map? %2) (into %1 (mbql-deps-map allow-int-ids? %2))+ (vector? %2) (into %1 (mbql-deps-vector allow-int-ids? %2)):else %1)#{}entity)))-(defn- mbql-deps-map [m]+(defn- mbql-deps-map [allow-int-ids? m](into #{}(mapcat (fn [[k v]](cond+ ;; --- serialized (portable) refs. Table/Field references contribute only their Database as a+ ;; dependency (see `ref->db-dep`); the referenced Table/Field are synthesized on import. ---(and (= k :database)(string? v)(not= v "database/__virtual")) #{[{:model "Database" :id v}]}- ;; Table/Field references contribute only their Database as a dependency (see `ref->db-dep`).(and (= k :source-table) (vector? v)) (ref->db-dep v)(and (= k :source-table) (portable-id? v)) #{[{:model "Card" :id v}]}(and (= k :source-card) (portable-id? v)) #{[{:model "Card" :id v}]}@@ -1497,19 +1542,32 @@(and (= k :snippet-id) (portable-id? v)) #{[{:model "NativeQuerySnippet" :id v}]}(and (= k :table-id) (vector? v)) (ref->db-dep v)(and (#{:card_id :card-id} k) (string? v)) #{[{:model "Card" :id v}]}- (map? v) (mbql-deps-map v)- (vector? v) (mbql-deps-vector v))))+ ;; --- raw (numeric) refs, walked at export time: the referenced Table/Field are real appdb ids to+ ;; existence-check. `allow-int-ids?` gates these (see `raw-ref-id?`). ---+ (and (= k :database) (raw-ref-id? allow-int-ids? v)) #{[{:model "Database" :id v}]}+ (and (= k :source-table) (raw-ref-id? allow-int-ids? v)) #{[{:model "Table" :id v}]}+ (and (= k :source-card) (raw-ref-id? allow-int-ids? v)) #{[{:model "Card" :id v}]}+ (and (= k :source-field) (raw-ref-id? allow-int-ids? v)) #{[{:model "Field" :id v}]}+ (and (= k :snippet-id) (raw-ref-id? allow-int-ids? v)) #{[{:model "NativeQuerySnippet" :id v}]}+ (and (= k :table-id) (raw-ref-id? allow-int-ids? v)) #{[{:model "Table" :id v}]}+ (and (#{:card_id :card-id} k) (raw-ref-id? allow-int-ids? v)) #{[{:model "Card" :id v}]}+ (map? v) (mbql-deps-map allow-int-ids? v)+ (vector? v) (mbql-deps-vector allow-int-ids? v))))m))(defn mbql-deps- "Given an MBQL expression as exported, with qualified names like `[\"some-db\" \"schema\" \"table_name\"]` instead of- raw IDs, return the corresponding set of serdes dependencies. The query can't be imported until all the referenced- databases, tables and fields are loaded."- [entity]+ "Given an MBQL expression, return the corresponding set of serdes dependencies. The query can't be imported until all+ the referenced databases, tables and fields are loaded.++ Pass `allow-int-ids?` true when `entity` is a raw appdb query whose references are still integer ids (export /+ serialization time) and you want those references collected as dependencies. Pass false when `entity` is in+ serialized/portable form (load time), where references are qualified names like `[\"some-db\" \"schema\"+ \"table_name\"]` and a bare integer is a stray value to ignore, not a reference."+ [allow-int-ids? entity](cond- (map? entity) (mbql-deps-map entity)- (seqable? entity) (mbql-deps-vector entity)- :else (mbql-deps-vector [entity])))+ (map? entity) (mbql-deps-map allow-int-ids? entity)+ (seqable? entity) (mbql-deps-vector allow-int-ids? entity)+ :else (mbql-deps-vector allow-int-ids? [entity])));;; ## Dashboard/Question Parameters@@ -1559,14 +1617,14 @@(defn parameters-deps"Given the :parameters (possibly nil) for an entity, return any embedded serdes-deps as a set.- Always returns an empty set even if the input is nil."- [parameters]+ Always returns an empty set even if the input is nil. For `allow-int-ids?` see [[mbql-deps]]."+ [allow-int-ids? parameters](reduce set/union #{}(for [parameter parameters:when (= (keyword (:values_source_type parameter)) :card):let [config (:values_source_config parameter)]](set/union #{[{:model "Card" :id (:card_id config)}]}- (mbql-deps-vector (:value_field config))))))+ (mbql-deps-vector allow-int-ids? (:value_field config))))));;; ## Viz settings@@ -1843,13 +1901,16 @@(update :column_settings import-column-settings))))(defn- viz-link-card-deps- [settings]+ [allow-int-ids? settings](when-let [{:keys [model id]} (get-in settings [:link :entity])]- #{(case model- ;; A linked Table is not a dependency (synthesized on import if missing), but its Database is.- "table" [{:model "Database" :id (first id)}]- [{:model (name (link-card-model->toucan-model model))- :id id}])}))+ (if (= model "table")+ ;; Serialized: a linked Table is not a dependency (synthesized on import), but its Database is. Raw (export+ ;; time): the numeric table id is a real Table to existence-check.+ (cond+ (vector? id) #{[{:model "Database" :id (first id)}]}+ (raw-ref-id? allow-int-ids? id) #{[{:model "Table" :id id}]})+ #{[{:model (name (link-card-model->toucan-model model))+ :id id}]})))(defn- viz-click-behavior-deps[settings]@@ -1865,21 +1926,21 @@(defn visualization-settings-deps"Given the :visualization_settings (possibly nil) for an entity, return any embedded serdes-deps as a set.- Always returns an empty set even if the input is nil."- [viz]+ Always returns an empty set even if the input is nil. For `allow-int-ids?` see [[mbql-deps]]."+ [allow-int-ids? viz](let [column-settings-keys-deps (some->> viz:column_settingskeys- (map (comp mbql-deps json/decode name)))+ (map (comp #(mbql-deps allow-int-ids? %) json/decode name)))column-settings-vals-deps (some->> viz:column_settingsvals(map viz-click-behavior-deps))- link-card-deps (viz-link-card-deps viz)+ link-card-deps (viz-link-card-deps allow-int-ids? viz)click-behavior-deps (viz-click-behavior-deps viz)](->> (concat column-settings-keys-depscolumn-settings-vals-deps- [(mbql-deps viz) link-card-deps click-behavior-deps])+ [(mbql-deps allow-int-ids? viz) link-card-deps click-behavior-deps])(filter some?)(reduce set/union #{}))))diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex fa0f1d31f48..b58d84b428b 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -190,11 +190,17 @@(when-let [collection_id (t2/select-one-fn :collection_id :model/NativeQuerySnippet :id id)]{["Collection" collection_id] {"NativeQuerySnippet" id}}))-(defmethod serdes/dependencies "NativeQuerySnippet"+(defmethod serdes/deserialization-dependencies "NativeQuerySnippet"[{:keys [collection_id]}](when collection_id[[{:model "Collection" :id collection_id}]]))+(defmethod serdes/serialization-dependencies "NativeQuerySnippet"+ [_model-name {:keys [collection_id]}]+ ;; A snippet only references its containing Collection, which a selective export may legitimately omit.+ (when collection_id+ #{[{:model "Collection" :id collection_id}]}))+(defmethod serdes/storage-path "NativeQuerySnippet" [snippet ctx](serdes/storage-default-collection-path snippet ctx "snippets"))diff --git a/src/metabase/osi/models/osi_ai_context.clj b/src/metabase/osi/models/osi_ai_context.cljindex 64fa6951526..ca76e0bfbf0 100644--- a/src/metabase/osi/models/osi_ai_context.clj+++ b/src/metabase/osi/models/osi_ai_context.clj@@ -159,7 +159,7 @@:import-with-context(fn [current _ _] (:entity_local_id (parent-path->entity (pop (serdes/path current)))))}}})-(defmethod serdes/dependencies "OsiAiContext"+(defmethod serdes/deserialization-dependencies "OsiAiContext"[entity];; Depend on the entity this row describes (its parent path), so it imports after that entity exists.[(vec (pop (serdes/path entity)))])diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex a9ed68c343e..3c627d2ff7e 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1343,10 +1343,10 @@;; FIXME: remove that `if` after v52(m/update-existing :fk_target_field_id #(if (number? %) % (serdes/*import-field-fk* %)))))))-(defn- result-metadata-deps [metadata]+(defn- result-metadata-deps [allow-int-ids? metadata](when (seq metadata)(-> (reduce into #{} (for [m metadata]- (serdes/mbql-deps (:field_ref m))))+ (serdes/mbql-deps allow-int-ids? (:field_ref m))))(disj nil))))(defmethod serdes/storage-path "Card" [card ctx]@@ -1418,24 +1418,28 @@:collection_preview true:enable_embedding false}})-(defmethod serdes/dependencies "Card"- [{:keys [collection_id database_id dataset_query parameters parameter_mappings- result_metadata source_card_id visualization_settings- dashboard_id document_id]}]+(defn- card-deps+ "The serdes dependencies of a Card as `:serdes/meta` paths. `allow-int-ids?` selects raw-appdb vs serialized ref semantics for+ the mbql walkers. Shared by [[serdes/deserialization-dependencies]] (allow-int-ids? false) and+ [[serdes/serialization-dependencies]] (allow-int-ids? true)."+ [allow-int-ids? {:keys [collection_id database_id dataset_query parameters parameter_mappings+ result_metadata source_card_id visualization_settings+ dashboard_id document_id]}](set(concat- (mapcat serdes/mbql-deps parameter_mappings)- (serdes/parameters-deps parameters)+ (mapcat #(serdes/mbql-deps allow-int-ids? %) parameter_mappings)+ (serdes/parameters-deps allow-int-ids? parameters)(when database_id [[{:model "Database" :id database_id}]])- ;; Note: `table_id` is intentionally not a dependency — the Database (above) is, and a missing Table is- ;; synthesized as an inactive row on import.(when source_card_id #{[{:model "Card" :id source_card_id}]})(when collection_id #{[{:model "Collection" :id collection_id}]})(when dashboard_id #{[{:model "Dashboard" :id dashboard_id}]})(when document_id #{[{:model "Document" :id document_id}]})- (result-metadata-deps result_metadata)- (serdes/mbql-deps dataset_query)- (serdes/visualization-settings-deps visualization_settings))))+ (result-metadata-deps allow-int-ids? result_metadata)+ (serdes/mbql-deps allow-int-ids? dataset_query)+ (serdes/visualization-settings-deps allow-int-ids? visualization_settings))))++(defmethod serdes/deserialization-dependencies "Card" [card]+ (card-deps false card))(defmethod serdes/descendants "Card" [_model-name id _opts](let [card (t2/select-one :model/Card :id id)@@ -1455,6 +1459,9 @@(for [snippet-id snippets]{["NativeQuerySnippet" snippet-id] {"Card" id}})))))+(defmethod serdes/serialization-dependencies "Card" [_model-name card]+ (card-deps true card))+;;;; ------------------------------------------------- Search ----------------------------------------------------------(mu/defn- dataset-query->dimensions :- [:maybe [:sequential ::lib.schema.metadata/column]]diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 431ffc41cde..66e34e9282e 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -217,8 +217,8 @@[_segment][:name (serdes/hydrated-hash :table) :created_at])-(defmethod serdes/dependencies "Segment" [{:keys [definition]}]- (serdes/mbql-deps definition))+(defmethod serdes/deserialization-dependencies "Segment" [{:keys [definition]}]+ (serdes/mbql-deps false definition))(defmethod serdes/storage-path "Segment" [segment _ctx](let [table-path (-> segment :definition serdes/serialized-query-source-table)]diff --git a/src/metabase/timeline/models/timeline.clj b/src/metabase/timeline/models/timeline.cljindex e3a79a99cba..ee67759a71a 100644--- a/src/metabase/timeline/models/timeline.clj+++ b/src/metabase/timeline/models/timeline.clj@@ -52,9 +52,14 @@[_timeline][:name (serdes/hydrated-hash :collection) :created_at])-(defmethod serdes/dependencies "Timeline" [{:keys [collection_id]}]+(defmethod serdes/deserialization-dependencies "Timeline" [{:keys [collection_id]}][[{:model "Collection" :id collection_id}]])+(defmethod serdes/serialization-dependencies "Timeline" [_model-name {:keys [collection_id]}]+ ;; A Timeline only references its containing Collection, which a selective export may legitimately omit.+ (when collection_id+ #{[{:model "Collection" :id collection_id}]}))+(defmethod serdes/make-spec "Timeline" [_model-name opts]{:copy [:archived :default :description :entity_id :icon :name]:skip []diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 0fd4827321b..5ad30e73ea8 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -470,7 +470,7 @@:import serdes/import-mbql}:tags (serdes/nested :model/TransformTransformTag :transform_id (merge {:sort-by (juxt :position :created_at)} opts))}})-(defmethod serdes/dependencies "Transform"+(defmethod serdes/deserialization-dependencies "Transform"[{:keys [collection_id source tags source_database_id]}](set(concat@@ -480,7 +480,7 @@[[{:model "Database" :id source_database_id}]])(for [{tag-id :tag_id} tags][{:model "TransformTag" :id tag-id}])- (serdes/mbql-deps source))))+ (serdes/mbql-deps false source))))(defmethod serdes/storage-path "Transform" [transform ctx](serdes/storage-default-collection-path transform ctx "transforms"))diff --git a/src/metabase/transforms/models/transform_job.clj b/src/metabase/transforms/models/transform_job.cljindex cd41a06a274..ba544235302 100644--- a/src/metabase/transforms/models/transform_job.clj+++ b/src/metabase/transforms/models/transform_job.clj@@ -222,7 +222,7 @@:created_at (serdes/date):job_tags (serdes/nested :model/TransformJobTransformTag :job_id (merge {:sort-by (juxt :position :created_at)} opts))}})-(defmethod serdes/dependencies "TransformJob"+(defmethod serdes/deserialization-dependencies "TransformJob"[{:keys [job_tags]}](set(for [{tag-id :tag_id} job_tags]diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 7c6a46cbbd7..69f2f7739e3 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -466,7 +466,7 @@field-q (serdes/recursively-find-field-q (:id table) (map :id (reverse fields)))](t2/select-one :model/Field field-q)))-(defmethod serdes/dependencies "Field" [field]+(defmethod serdes/deserialization-dependencies "Field" [field](let [db-path (first (serdes/path field))]#{[db-path]}))diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex 08bb3a53ee6..48068517f31 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -44,7 +44,7 @@(conj (serdes/generate-path "Field" {:id field_id}){:model "FieldUserSettings" :id "1"}))-(defmethod serdes/dependencies "FieldUserSettings" [fv]+(defmethod serdes/deserialization-dependencies "FieldUserSettings" [fv](let [db-path (first (serdes/path fv))][[db-path]]))diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 39e3c2c0380..85827de54c7 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -559,7 +559,7 @@(conj (serdes/generate-path "Field" {:id field_id}){:model "FieldValues" :id "0"}))-(defmethod serdes/dependencies "FieldValues" [fv]+(defmethod serdes/deserialization-dependencies "FieldValues" [fv](let [db-path (first (serdes/path fv))][[db-path]]))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex a264ea6d46a..3079c618929 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -583,7 +583,7 @@(t2/select-one :model/Database :id (:db_id table)));;; ------------------------------------------------- Serialization --------------------------------------------------(defmethod serdes/dependencies "Table" [{:keys [db_id collection_id transform_id]}]+(defmethod serdes/deserialization-dependencies "Table" [{:keys [db_id collection_id transform_id]}](cond-> [[{:model "Database" :id db_id}]]collection_id (conj [{:model "Collection" :id collection_id}])transform_id (conj [{:model "Transform" :id transform_id}])))diff --git a/test/metabase/documents/models/document_test.clj b/test/metabase/documents/models/document_test.cljindex 75da7c6d203..6e882b19e30 100644--- a/test/metabase/documents/models/document_test.clj+++ b/test/metabase/documents/models/document_test.clj@@ -448,7 +448,7 @@(let [document {:collection_id 123:creator_id 456:serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (= #{[{:model "Collection" :id 123}]}deps))))))@@ -463,7 +463,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 789}]))(is (contains? deps [{:model "Collection" :id 123}])))))@@ -484,7 +484,7 @@:model "table"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 789}]))(is (contains? deps [{:model "Dashboard" :id 456}]))(is (contains? deps [{:model "Table" :id 321}]))@@ -505,7 +505,7 @@:model "card"}}]}]}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 999}]))(is (contains? deps [{:model "Collection" :id 123}])))))@@ -518,7 +518,7 @@:content [{:type "text" :text "Plain text only"}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (= #{[{:model "Collection" :id 123}]}deps)))))@@ -536,7 +536,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 456}]))(is (not (some #(= (:model (first %)) "unknown-model") deps))))))@@ -553,7 +553,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 456}]))(is (= 2 (count deps)))))) ; collection and one valid card@@ -571,7 +571,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 789}]))(is (= 2 (count deps)))))) ; collection and one unique card@@ -583,7 +583,7 @@:content []}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (= #{[{:model "Collection" :id 123}]}deps)))))@@ -600,7 +600,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 456}]))(is (= 2 (count deps)))))) ; collection and one valid card@@ -617,7 +617,7 @@:model "card"}}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 456}]))(is (= 2 (count deps)))))) ; collection and one valid card@@ -647,7 +647,7 @@:model "table"}}]}]}]}]}]}:content_type "application/json+vnd.prose-mirror":serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Card" :id 789}]))(is (contains? deps [{:model "Dashboard" :id 456}]))(is (contains? deps [{:model "Table" :id 321}]))@@ -664,7 +664,7 @@:model "card"}}]}:content_type "application/json" ; Not prose-mirror:serdes/meta [{:model "Document" :id "test-entity-id"}]}- deps (serdes/dependencies document)]+ deps (serdes/deserialization-dependencies document)](is (contains? deps [{:model "Collection" :id 123}])))))(deftest document-serdes-descendants-embedded-cards-testdiff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex b574e15f6d6..39d1cd775cd 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -265,7 +265,36 @@(testing "template tag :table-id contributes only its Database dependency — the referenced Table itself is not adependency (it's synthesized on import if missing)"(is (= #{[{:model "Database" :id "DB"}]}- (#'serdes/mbql-deps-map {:table-id ["DB" "SCHEMA" "TABLE"]})))))+ (#'serdes/mbql-deps-map false {:table-id ["DB" "SCHEMA" "TABLE"]})))))++(deftest ^:parallel mbql-deps-format-parity-test+ (testing "mbql-deps finds each reference on both the serialized (portable) and the raw (numeric) form of a query.+ serialization-dependencies runs on raw entities and existence-checks the referenced Table/Field;+ deserialization-dependencies runs on the serialized form, where Table/Field are synthesized on import, so+ it reports only their Database. Every other ref type resolves to the same model in both forms. This is the+ parity guard for the two dependency codepaths sharing mbql-deps."+ (let [models (fn [deps] (into #{} (map (comp :model last)) deps))+ eid (fn [c] (apply str (repeat 21 c)))]+ (testing "MBQL ref clauses"+ (doseq [[label serialized raw ser-models raw-models]+ [["field (pMBQL)" [:field {} ["DB" "S" "T" "F"]] [:field {} 53] #{"Database"} #{"Field"}]+ ["field (legacy)" [:field ["DB" "S" "T" "F"] {}] [:field 53 {}] #{"Database"} #{"Field"}]+ ["metric" [:metric {} (eid \a)] [:metric {} 99] #{"Card"} #{"Card"}]+ ["segment" [:segment {} (eid \b)] [:segment {} 5] #{"Segment"} #{"Segment"}]+ ["measure" [:measure {} (eid \c)] [:measure {} 3] #{"Measure"} #{"Measure"}]]]+ (testing label+ (is (= ser-models (models (serdes/mbql-deps false serialized))) "serialized (portable) form")+ (is (= raw-models (models (serdes/mbql-deps true raw)))+ "raw (numeric) form"))))+ (testing "MBQL map keys"+ (doseq [[label serialized raw ser-models raw-models]+ [["source-table" {:source-table ["DB" "S" "T"]} {:source-table 9} #{"Database"} #{"Table"}]+ ["source-card" {:source-card (eid \d)} {:source-card 7} #{"Card"} #{"Card"}]+ ["snippet-id" {:snippet-id (eid \e)} {:snippet-id 2} #{"NativeQuerySnippet"} #{"NativeQuerySnippet"}]]]+ (testing label+ (is (= ser-models (models (#'serdes/mbql-deps-map false serialized))) "serialized")+ (is (= raw-models (models (#'serdes/mbql-deps-map true raw)))+ "raw")))))))(deftest ^:parallel export-parameters-test(binding [serdes/*export-fk* (fn [id model]