Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Name the referring entity in remote sync not-found errors (#77430)

Eric Normand5a4f82063e80
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index eda5cd37bcf..57402f35c97 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -82,50 +82,114 @@
;; No predicate (global model, no imported ids or conditions) — delete all
:else (t2/delete! model-key))))
+(defn- quoted
+ "Wraps `s` in backticks so that leading and trailing whitespace is visible to the reader."
+ [s]
+ (str "`" s "`"))
+
+(defn- describe-entity
+ "Renders `{:model :id :name}` as e.g. ``Card `Orders by Month` (`abc123`)``. Omits whichever of name/id is nil."
+ [{:keys [model id] entity-name :name}]
+ (cond-> (or model "Content")
+ entity-name (str " " (quoted entity-name))
+ id (str " (" (quoted id) ")")))
+
+(defn- missing-reference-message
+ "Renders the user-facing message for an import that references content absent from this instance.
+
+ `missing` is `{:model :id :name}` describing the content that could not be found; `referrer` is the same shape
+ for the entity holding the dangling reference, and may be nil when it is unknown. `:name` may be nil on either."
+ [{:keys [missing referrer]}]
+ (format "Import failed: %s does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."
+ (if referrer
+ (format "%s references %s, which" (describe-entity referrer) (describe-entity missing))
+ (describe-entity missing))))
+
+(defn- sentence
+ "Terminates `s` with a period unless it already ends with one. Returns nil for blank input."
+ [s]
+ (when (seq s)
+ (cond-> s
+ (not (str/ends-with? s ".")) (str "."))))
+
+(defn- load-failure-message
+ "Renders the user-facing message for content that could not be written to the appdb.
+
+ `entity` is `{:model :id :name}` describing the content that failed; `reason` is the underlying error text, and
+ may be nil. `stripped-keys`, when present, names the keys that were removed to break a circular dependency —
+ a partial row for `entity` may have been committed."
+ [{:keys [entity reason stripped-keys]}]
+ (->> [(format "Import failed: could not save %s." (describe-entity entity))
+ (sentence reason)
+ (when (seq stripped-keys)
+ (format "It may have been saved without: %s."
+ (str/join ", " (map (comp quoted name) (sort stripped-keys)))))]
+ (remove nil?)
+ (str/join " ")))
+
+(defn- cause-with-error
+ "Returns the first exception in `e`'s cause chain whose ex-data `:error` is `error-type`, or nil."
+ [e error-type]
+ (->> (iterate ex-cause e)
+ (take-while some?)
+ (some (fn [ex]
+ (when (= error-type (:error (ex-data ex)))
+ ex)))))
+
(defn source-error-message
"Constructs user-friendly error messages from remote sync source exceptions.
Takes a throwable exception and returns a string message that categorizes the error (network, authentication,
repository not found, branch, or generic) based on the exception type and message content."
[e]
- (cond
- (or (instance? java.net.UnknownHostException e)
- (instance? java.net.UnknownHostException (ex-cause e)))
- "Network error: Unable to reach git repository host"
-
- (str/includes? (ex-message e) "Authentication failed")
- "Authentication failed: Please check your git credentials"
-
- (str/includes? (ex-message e) "Repository not found")
- "Repository not found: Please check the repository URL"
-
- (str/includes? (ex-message e) "branch")
- "Branch error: Please check the specified branch exists"
-
- (some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))
- "This repository contains tenant collections, but the tenants feature is disabled on your instance."
-
- (str/includes? (ex-message e) "Missing commit")
- "Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."
-
- (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)
- (let [{:keys [model id]} (ex-data e)]
- (format "Import failed: %s '%s' does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing." model id))
-
- (some-> e ex-cause ex-message (str/includes? "database not found"))
- (format "Import failed: A referenced database does not exist on this instance. %s" (ex-message (ex-cause e)))
-
- (seq (:ingest-errors (ex-data e)))
- (let [ingest-errors (:ingest-errors (ex-data e))]
- (format "Failed to read %d file(s) from the repository: %s"
- (count ingest-errors)
- (str/join "; " (for [ie ingest-errors
- :let [{:keys [file reason]} (ex-data ie)]]
- (cond-> file
- reason (str ": " reason))))))
+ (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)]
+ (cond
+ (or (instance? java.net.UnknownHostException e)
+ (instance? java.net.UnknownHostException (ex-cause e)))
+ "Network error: Unable to reach git repository host"
+
+ (str/includes? (ex-message e) "Authentication failed")
+ "Authentication failed: Please check your git credentials"
+
+ (str/includes? (ex-message e) "Repository not found")
+ "Repository not found: Please check the repository URL"
+
+ (str/includes? (ex-message e) "branch")
+ "Branch error: Please check the specified branch exists"
+
+ (some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))
+ "This repository contains tenant collections, but the tenants feature is disabled on your instance."
+
+ (str/includes? (ex-message e) "Missing commit")
+ "Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."
+
+ (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)
+ (let [{:keys [model id referrer]} (ex-data e)]
+ (missing-reference-message {:missing {:model model :id id}
+ :referrer referrer}))
+
+ ;; the entity that failed to load is the one holding the reference to the absent database
+ missing-db
+ (missing-reference-message {:missing {:model "Database" :id (:db-name (ex-data missing-db))}
+ :referrer (:entity (ex-data e))})
+
+ (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/load-failure)
+ (let [{:keys [entity stripped-keys]} (ex-data e)]
+ (load-failure-message {:entity entity
+ :reason (some-> e ex-cause ex-message)
+ :stripped-keys stripped-keys}))
+
+ (seq (:ingest-errors (ex-data e)))
+ (let [ingest-errors (:ingest-errors (ex-data e))]
+ (format "Failed to read %d file(s) from the repository: %s"
+ (count ingest-errors)
+ (str/join "; " (for [ie ingest-errors
+ :let [{:keys [file reason]} (ex-data ie)]]
+ (cond-> (quoted file)
+ reason (str ": " reason))))))
- :else
- (format "Failed to reload from git repository: %s" (ex-message e))))
+ :else
+ (format "Failed to reload from git repository: %s" (ex-message e)))))
(defn- get-conflicts
"Detects conflicts that would prevent or complicate import. Returns a map with two classes:
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index d9cad571a2e..1045ce00925 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -100,6 +100,30 @@
:table (some->> last-model (keyword "model") t2/table-name)
:error error-type}))
+(defn- entity-reference
+ "Describes the entity at `path` as `{:model :id :name}` for use in user-facing error messages.
+ `:name` is nil for models whose name is absent or not a plain string (e.g. Settings, templated Card names)."
+ [path ingested]
+ (let [self (last path)]
+ {:model (:model self)
+ :id (:id self)
+ :name (when (string? (:name ingested))
+ (:name ingested))}))
+
+(defn- rethrow-with-referrer
+ "Always throws. Rethrows `e` with `referrer` attached when it is a `::not-found` error that does not name one yet,
+ and unchanged otherwise.
+
+ Only the nearest enclosing [[load-one!]] attaches itself, since it unwinds first and later frames see the key
+ already present. The replacement inherits `e`'s cause rather than nesting `e` beneath itself: error reporting
+ renders the whole cause chain, so nesting would print the same message twice joined by `caused by:`."
+ [e referrer]
+ (let [data (ex-data e)]
+ (if (and (= ::not-found (:error data))
+ (not (contains? data :referrer)))
+ (throw (ex-info (ex-message e) (assoc data :referrer referrer) (ex-cause e)))
+ (throw e))))
+
(defn- valid-model-name-for-load? [model-name]
;; linear scan, but small n
(->> (concat serdes.models/inlined-models
@@ -179,11 +203,14 @@
{:entity_id (:entity_id ingested)
:level (count expanding)
:deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))
- ctx (-> ctx
- (update :expanding conj path)
- (load-deps! deps)
- (update :seen conj path)
- (update :expanding disj path))
+ ctx (try
+ (-> ctx
+ (update :expanding conj path)
+ (load-deps! deps)
+ (update :seen conj path)
+ (update :expanding disj path))
+ (catch Exception e
+ (rethrow-with-referrer e (entity-reference rebuilt-path ingested))))
_ (when (seq deps)
(log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested)
:level (count expanding)}))
@@ -205,8 +232,9 @@
(format " (it may have been left without these keys, which were stripped to break a circular dependency: %s)"
(str/join ", " (sort (map name (keys-to-strip ingested)))))
""))
- (cond-> (path-error-data ::load-failure expanding path)
- stripped? (assoc :stripped-keys (keys-to-strip ingested)))
+ (-> (path-error-data ::load-failure expanding path)
+ (assoc :entity (entity-reference rebuilt-path ingested))
+ (cond-> stripped? (assoc :stripped-keys (keys-to-strip ingested))))
e))))))))))
(defn new-context
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index fc15d5a598a..8fee1791581 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -104,15 +104,104 @@
:model "Database"
:id "clickhouse"
:error :metabase-enterprise.serialization.v2.load/not-found})]
- (is (= "Import failed: Database 'clickhouse' does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."
+ (is (= "Import failed: Database (`clickhouse`) does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."
(impl/source-error-message e)))))
- (testing "source-error-message produces helpful message for FK database-not-found errors"
+ (testing "source-error-message names the entity holding the dangling reference when known (GHY-3992)"
+ (let [e (ex-info "Collection 'xyz789' was not found"
+ {:path "Collection xyz789"
+ :model "Collection"
+ :id "xyz789"
+ :referrer {:model "Card" :id "abc123" :name "Orders by Month"}
+ :error :metabase-enterprise.serialization.v2.load/not-found})]
+ (is (= (str "Import failed: Card `Orders by Month` (`abc123`) references Collection (`xyz789`), which does not "
+ "exist on this instance. Make sure all referenced databases and other dependencies are set up "
+ "before importing.")
+ (impl/source-error-message e)))))
+ (testing "surrounding whitespace in a referenced name is visible because names are backtick-quoted (GHY-3992)"
+ (doseq [db-name ["My Database " " My Database" "My Database"]]
+ (let [e (ex-info (format "Database '%s' was not found" db-name)
+ {:path (str "Database " db-name)
+ :model "Database"
+ :id db-name
+ :error :metabase-enterprise.serialization.v2.load/not-found})]
+ (is (str/includes? (impl/source-error-message e)
+ (format "Database (`%s`)" db-name))
+ (format "expected %s to be quoted verbatim" (pr-str db-name)))))))
+
+(deftest source-error-message-database-not-found-test
+ (testing "source-error-message names the card and the missing database for FK database-not-found errors"
(let [cause (ex-info "table id present, but database not found: [clickhouse nil some_table]"
- {:table-id ["clickhouse" nil "some_table"]})
+ {:table-id ["clickhouse" nil "some_table"]
+ :db-name "clickhouse"
+ :error :metabase.models.serialization.resolve.db/database-not-found})
+ e (ex-info "Failed to load into database for Card abc123"
+ {:path "Card abc123"
+ :entity {:model "Card" :id "abc123" :name "Some card"}}
+ cause)]
+ (is (= (str "Import failed: Card `Some card` (`abc123`) references Database (`clickhouse`), which does not "
+ "exist on this instance. Make sure all referenced databases and other dependencies are set up "
+ "before importing.")
+ (impl/source-error-message e)))))
+ (testing "database-not-found is found anywhere in the cause chain, not only at the immediate cause"
+ (let [root (ex-info "table id present, but database not found: [clickhouse nil t]"
+ {:db-name "clickhouse"
+ :error :metabase.models.serialization.resolve.db/database-not-found})
+ middle (ex-info "wrapped by an intervening helper" {} root)
+ e (ex-info "Failed to load into database for Card abc123" {:path "Card abc123"} middle)]
+ (is (str/includes? (impl/source-error-message e) "Database (`clickhouse`)")))))
+
+(deftest source-error-message-load-failure-test
+ (testing "source-error-message names the entity and the underlying reason (GHY-3992)"
+ (let [cause (ex-info "NOT NULL constraint failed: report_card.display" {})
e (ex-info "Failed to load into database for Card abc123"
- {:path "Card abc123"}
+ {:path "Card abc123"
+ :entity {:model "Card" :id "abc123" :name "Orders by Month"}
+ :error :metabase-enterprise.serialization.v2.load/load-failure}
+ cause)]
+ (is (= (str "Import failed: could not save Card `Orders by Month` (`abc123`). "
+ "NOT NULL constraint failed: report_card.display.")
+ (impl/source-error-message e)))))
+ (testing "a reason that already ends in a period is not double-punctuated"
+ (let [e (ex-info "Failed to load into database for Card abc123"
+ {:entity {:model "Card" :id "abc123" :name "Orders by Month"}
+ :error :metabase-enterprise.serialization.v2.load/load-failure}
+ (ex-info "Something went wrong." {}))]
+ (is (str/ends-with? (impl/source-error-message e) "went wrong."))))
+ (testing "a load-failure with no cause omits the reason clause"
+ (let [e (ex-info "Failed to load into database for Card abc123"
+ {:entity {:model "Card" :id "abc123" :name "Orders by Month"}
+ :error :metabase-enterprise.serialization.v2.load/load-failure})]
+ (is (= "Import failed: could not save Card `Orders by Month` (`abc123`)."
+ (impl/source-error-message e)))))
+ (testing "stripped keys are reported, since a partial row may have been committed (GHY-3992)"
+ (let [cause (ex-info "some db error" {})
+ e (ex-info "Failed to load into database for Dashboard xyz"
+ {:path "Dashboard xyz"
+ :entity {:model "Dashboard" :id "xyz" :name "Sales"}
+ :stripped-keys #{:parameters :dashcards}
+ :error :metabase-enterprise.serialization.v2.load/load-failure}
cause)]
- (is (str/includes? (impl/source-error-message e) "A referenced database does not exist on this instance"))))
+ (is (= (str "Import failed: could not save Dashboard `Sales` (`xyz`). some db error. "
+ "It may have been saved without: `dashcards`, `parameters`.")
+ (impl/source-error-message e)))))
+ (testing "a database-not-found cause still wins over the generic load-failure branch"
+ (let [cause (ex-info "table id present, but database not found: [ch nil t]"
+ {:db-name "ch"
+ :error :metabase.models.serialization.resolve.db/database-not-found})
+ e (ex-info "Failed to load into database for Card abc123"
+ {:entity {:model "Card" :id "abc123" :name "Some card"}
+ :error :metabase-enterprise.serialization.v2.load/load-failure}
+ cause)]
+ (is (str/includes? (impl/source-error-message e) "references Database (`ch`)"))))
+ (testing "a tenant-collection cause still wins over the generic load-failure branch"
+ (let [cause (ex-info "Can't create a tenant collection without tenants enabled" {})
+ e (ex-info "Failed to load into database for Collection abc"
+ {:entity {:model "Collection" :id "abc"}
+ :error :metabase-enterprise.serialization.v2.load/load-failure}
+ cause)]
+ (is (str/includes? (impl/source-error-message e) "tenants feature is disabled")))))
+
+(deftest source-error-message-ingest-errors-test
(testing "source-error-message lists each unreadable file with its parse reason (GHY-3887)"
(let [ingest-err (ex-info "Failed to parse file: collections/transforms/a.yaml"
{:file "collections/transforms/a.yaml"
@@ -123,8 +212,23 @@
ingest-err)
msg (impl/source-error-message e)]
(is (str/includes? msg "Failed to read 1 file(s)"))
- (is (str/includes? msg "collections/transforms/a.yaml"))
- (is (str/includes? msg "found character '@'")))))
+ (is (str/includes? msg "`collections/transforms/a.yaml`"))
+ (is (str/includes? msg "found character '@'"))))
+ (testing "file paths are backtick-quoted so surrounding whitespace is visible (GHY-3992)"
+ (let [ingest-err (ex-info "Failed to read file: collections/bar .yaml"
+ {:file "collections/bar .yaml"
+ :reason "IOException"})
+ e (ex-info "Failed to read 1 file(s) during ingestion: collections/bar .yaml"
+ {:ingest-errors [ingest-err]}
+ ingest-err)]
+ (is (= "Failed to read 1 file(s) from the repository: `collections/bar .yaml`: IOException"
+ (impl/source-error-message e)))))
+ (testing "each unreadable file is quoted independently when several fail"
+ (let [errs [(ex-info "a" {:file "a.yaml" :reason "bad"})
+ (ex-info "b" {:file "b.yaml" :reason "worse"})]
+ e (ex-info "Failed to read 2 file(s) during ingestion" {:ingest-errors errs} (first errs))]
+ (is (= "Failed to read 2 file(s) from the repository: `a.yaml`: bad; `b.yaml`: worse"
+ (impl/source-error-message e))))))
;; We need to make sure the task-id we use to track the Remote Sync is not bound to a transactions because of the behavior of
;; update-sync-progress. So the follow two tests cannot use with-temp to create models
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index d2d5b122048..e64a8bb9e96 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -1246,10 +1246,58 @@
:entity_id "0123456789abcdef_0123"
:name "Some card"
:table_id ["bad-db" nil "CUSTOMERS"]
- :visualization_settings {}}])]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo
- #"was not found"
- (serdes.load/load-metabase! ingestion)))))))))
+ :visualization_settings {}}])
+ e (try
+ (serdes.load/load-metabase! ingestion)
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (some? e))
+ (is (re-find #"was not found" (ex-message e)))
+ (testing "the not-found error names the entity holding the dangling reference (GHY-3992)"
+ (is (= {:model "Database" :id "bad-db"}
+ (select-keys (ex-data e) [:model :id])))
+ (is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}
+ (:referrer (ex-data e)))))
+ ;; attaching the referrer must not nest the original beneath itself: error reporting renders the
+ ;; whole cause chain, and a self-nested exception prints the same message twice
+ (testing "attaching the referrer does not duplicate the error through the cause chain"
+ (is (nil? (ex-cause e))))))
+ ;; `result-metadata-deps` derives deps only from each entry's `:field_ref`, never from its `:table_id`.
+ ;; So a result_metadata `:table_id` naming an absent database gets past dependency resolution and only
+ ;; fails inside `serdes/load-one!`, where `import-table-fk` raises ::database-not-found. (The Card's own
+ ;; top-level `table_id` cannot be used here: `populate-query-fields` recomputes it from `dataset_query`.)
+ (testing "a result_metadata table_id naming an absent database names the database and the card (GHY-3992)"
+ (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
+ :created_at (t/instant)
+ :creator_id "[email protected]"
+ :database_id "my-db"
+ :dataset_query {:database "my-db"
+ :type :query
+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}
+ :display :table
+ :entity_id "0123456789abcdef_0123"
+ :name "Some card"
+ :result_metadata [{:name "STATE"
+ :base_type :type/Text
+ :table_id ["absent-db" nil "CUSTOMERS"]}]
+ :table_id ["my-db" nil "CUSTOMERS"]
+ :visualization_settings {}}])
+ e (try
+ (serdes.load/load-metabase! ingestion)
+ nil
+ (catch clojure.lang.ExceptionInfo e e))
+ root (->> (iterate ex-cause e)
+ (take-while some?)
+ (some #(when (= :metabase.models.serialization.resolve.db/database-not-found
+ (:error (ex-data %)))
+ %)))]
+ (is (some? e))
+ (testing "the root cause carries the missing database name"
+ (is (some? root))
+ (is (= "absent-db" (:db-name (ex-data root)))))
+ (testing "the outer load-failure names the card that failed to load"
+ (is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}
+ (:entity (ex-data e)))))))))))
(deftest card-with-snippet-test
(let [db1s (atom nil)
diff --git a/src/metabase/models/serialization/resolve/db.clj b/src/metabase/models/serialization/resolve/db.clj
index d8ab17369c8..813a85c66d4 100644
--- a/src/metabase/models/serialization/resolve/db.clj
+++ b/src/metabase/models/serialization/resolve/db.clj
@@ -126,8 +126,10 @@
(or (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id db-id)
(synthesize-table! db-id schema table-name))
(throw (ex-info (format "table id present, but database not found: %s" table-id)
- {:table-id table-id
- :database-names (sort (t2/select-fn-vec :name :model/Table))})))))
+ {:table-id table-id
+ :db-name db-name
+ :database-names (sort (t2/select-fn-vec :name :model/Database))
+ :error ::database-not-found})))))
(defn import-field-fk
"Given [db-name schema table-name field-name ...], return numeric field_id. If part of the parent