Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix circular dependency error for cards with self-referencing parameters (#75731)

Eric Normandaff4a39ab730
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index f6ebed38890..32092244269 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -42,12 +42,12 @@
pointing to the dashboard it's in. But when we try to load that dashboard, we'll create all its dashcards, and one
of those dashcards will point to the card we started with.
- This map works around this: given a model (e.g. `Card`) that triggered a dependency loop, it provides a set of paths to
- keys to remove from the model so that we'll be able to successfully load it. You can remove keys in vectors using :* to
- indicate that all items in that vector should have a key removed."
- {"Dashboard" #{:dashcards}
+ This map works around this: given a model (e.g. `Card`) that triggered a dependency loop, it provides the set of
+ top-level keys to remove from the model so that we'll be able to successfully load it. The stripped keys are restored
+ when the original (outer) load of the entity completes its full pass."
+ {"Dashboard" #{:dashcards :parameters}
"Document" #{:document}
- "Card" #{:dashboard_id :document_id}})
+ "Card" #{:dashboard_id :document_id :parameters}})
(def ^:private ^:dynamic *warned-version-mismatch*
"Used to avoid double-logging on version mismatches. Warns if nil or set to true"
@@ -220,10 +220,19 @@
(serdes/load-one! ingested local-or-nil))))
ctx
(catch Exception e
- ;; ugly mapv here to convert #ordered/map into normal map so it's readable in the logs
- (throw (ex-info (format "Failed to load into database for %s" (serdes/log-path-str path))
- (path-error-data ::load-failure expanding path)
- e)))))))))
+ ;; if the entity was part of a dependency loop, a stripped version of it may already be committed; with
+ ;; continue-on-error that stripped row survives the import, so leave a breadcrumb in the error
+ (let [stripped? (contains? (:circular ctx) path)]
+ ;; ugly mapv here to convert #ordered/map into normal map so it's readable in the logs
+ (throw (ex-info (format "Failed to load into database for %s%s"
+ (serdes/log-path-str path)
+ (if stripped?
+ (format " (it may have been left without these keys, which were stripped to break a circular dependency: %s)"
+ (str/join ", " (sort (map name (keys-to-strip ingested)))))
+ ""))
+ (cond-> (path-error-data ::load-failure expanding path)
+ stripped? (assoc :stripped-keys (keys-to-strip ingested)))
+ e))))))))))
(defn new-context
"Given an ingestion create a new context for serialization.
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index b3cb83947c7..7677f6445bd 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -1670,6 +1670,94 @@
(is (= (t2/select-one-fn :id :model/Dashboard :entity_id (:entity_id dash1))
(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id card-2))))))))))
+(defn- card-sourced-param
+ "A category parameter whose dropdown values come from `card-id`'s results."
+ [card-id]
+ {:id "abc"
+ :type "category"
+ :name "CATEGORY"
+ :slug "category"
+ :values_source_type "card"
+ :values_source_config {:card_id card-id}})
+
+(deftest self-referencing-parameter-card-test
+ (testing "a card whose parameter sources dropdown values from the card itself can be loaded (#73133)"
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [coll (ts/create! :model/Collection :name "coll")
+ card (ts/create! :model/Card :name "self-ref card" :collection_id (:id coll))
+ _ (t2/update! :model/Card (:id card)
+ {:parameters [(card-sourced-param (:id card))]})
+ ser (into [] (serdes.extract/extract {:no-settings true
+ :no-data-model false}))]
+ (testing "loading on top of the existing card"
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (= (:id card)
+ (-> (t2/select-one :model/Card :entity_id (:entity_id card))
+ :parameters first :values_source_config :card_id))))
+ (testing "loading into an empty database"
+ (ts/with-db dest-db
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (let [new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
+ (is (= (:id new-card)
+ (-> new-card :parameters first :values_source_config :card_id)))))))))))
+
+(deftest mutually-referencing-parameter-cards-test
+ (testing "two cards whose parameters source dropdown values from each other can be loaded (#73133)"
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [coll (ts/create! :model/Collection :name "coll")
+ card-a (ts/create! :model/Card :name "card a" :collection_id (:id coll))
+ card-b (ts/create! :model/Card :name "card b" :collection_id (:id coll))
+ _ (t2/update! :model/Card (:id card-a) {:parameters [(card-sourced-param (:id card-b))]})
+ _ (t2/update! :model/Card (:id card-b) {:parameters [(card-sourced-param (:id card-a))]})
+ ser (into [] (serdes.extract/extract {:no-settings true
+ :no-data-model false}))]
+ (testing "loading on top of the existing cards"
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (= (:id card-b)
+ (-> (t2/select-one :model/Card :entity_id (:entity_id card-a))
+ :parameters first :values_source_config :card_id)))
+ (is (= (:id card-a)
+ (-> (t2/select-one :model/Card :entity_id (:entity_id card-b))
+ :parameters first :values_source_config :card_id))))
+ (testing "loading into an empty database"
+ (ts/with-db dest-db
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (let [new-a (t2/select-one :model/Card :entity_id (:entity_id card-a))
+ new-b (t2/select-one :model/Card :entity_id (:entity_id card-b))]
+ (is (= (:id new-b)
+ (-> new-a :parameters first :values_source_config :card_id)))
+ (is (= (:id new-a)
+ (-> new-b :parameters first :values_source_config :card_id)))))))))))
+
+(deftest dashboard-parameter-sourcing-own-dashboard-question-test
+ (testing "a dashboard whose parameter sources dropdown values from a dashboard question on that same dashboard can be loaded (#73133)"
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [coll (ts/create! :model/Collection :name "coll")
+ dash (ts/create! :model/Dashboard :name "dash" :collection_id (:id coll))
+ card (ts/create! :model/Card :name "dq card" :dashboard_id (:id dash))
+ _ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id card))
+ _ (t2/update! :model/Dashboard (:id dash)
+ {:parameters [(card-sourced-param (:id card))]})
+ ser (into [] (serdes.extract/extract {:no-settings true
+ :no-data-model false}))]
+ (testing "loading on top of the existing dashboard"
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (= (:id card)
+ (-> (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
+ :parameters first :values_source_config :card_id))))
+ (testing "loading into an empty database"
+ (ts/with-db dest-db
+ (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (let [new-dash (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
+ new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
+ (is (= (:id new-card)
+ (-> new-dash :parameters first :values_source_config :card_id)))
+ (is (= (:id new-dash)
+ (:dashboard_id new-card)))))))))))
+
(deftest continue-on-error-test
(let [change-ser (fn [ser changes] ;; kind of like left-join, but right side is indexed
(vec (for [entity ser]
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index f2ace67a3ee..3ee45872063 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -858,7 +858,8 @@
(defn- xform-one [model-name ingested]
(let [spec (*make-spec* model-name nil)]
(assert spec (str "No serialization spec defined for model " model-name))
- (-> (merge (:defaults spec) (select-keys ingested (:copy spec)))
+ (-> (merge (:defaults spec)
+ (select-keys (apply dissoc ingested (::strip ingested)) (:copy spec)))
(into (for [[k transform] (:transform spec)
:when (and (not (::nested transform))
;; handling circuit-breaking
@@ -873,7 +874,8 @@
(or (some? res)
(contains? ingested import-k)))]
[k res]))
- (coerce-keys (:coerce spec)))))
+ ;; coercing a stripped key would re-introduce it as an explicit nil
+ (coerce-keys (apply dissoc (:coerce spec) (::strip ingested))))))
(defn- spec-nested! [model-name ingested instance]
(let [spec (*make-spec* model-name nil)]