The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Name the referring entity in remote sync not-found errors (#77430)
Eric Normand5a4f82063e80
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex eda5cd37bcf..57402f35c97 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -82,50 +82,114 @@;; No predicate (global model, no imported ids or conditions) — delete all:else (t2/delete! model-key))))+(defn- quoted+ "Wraps `s` in backticks so that leading and trailing whitespace is visible to the reader."+ [s]+ (str "`" s "`"))++(defn- describe-entity+ "Renders `{:model :id :name}` as e.g. ``Card `Orders by Month` (`abc123`)``. Omits whichever of name/id is nil."+ [{:keys [model id] entity-name :name}]+ (cond-> (or model "Content")+ entity-name (str " " (quoted entity-name))+ id (str " (" (quoted id) ")")))++(defn- missing-reference-message+ "Renders the user-facing message for an import that references content absent from this instance.++ `missing` is `{:model :id :name}` describing the content that could not be found; `referrer` is the same shape+ for the entity holding the dangling reference, and may be nil when it is unknown. `:name` may be nil on either."+ [{:keys [missing referrer]}]+ (format "Import failed: %s does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."+ (if referrer+ (format "%s references %s, which" (describe-entity referrer) (describe-entity missing))+ (describe-entity missing))))++(defn- sentence+ "Terminates `s` with a period unless it already ends with one. Returns nil for blank input."+ [s]+ (when (seq s)+ (cond-> s+ (not (str/ends-with? s ".")) (str "."))))++(defn- load-failure-message+ "Renders the user-facing message for content that could not be written to the appdb.++ `entity` is `{:model :id :name}` describing the content that failed; `reason` is the underlying error text, and+ may be nil. `stripped-keys`, when present, names the keys that were removed to break a circular dependency —+ a partial row for `entity` may have been committed."+ [{:keys [entity reason stripped-keys]}]+ (->> [(format "Import failed: could not save %s." (describe-entity entity))+ (sentence reason)+ (when (seq stripped-keys)+ (format "It may have been saved without: %s."+ (str/join ", " (map (comp quoted name) (sort stripped-keys)))))]+ (remove nil?)+ (str/join " ")))++(defn- cause-with-error+ "Returns the first exception in `e`'s cause chain whose ex-data `:error` is `error-type`, or nil."+ [e error-type]+ (->> (iterate ex-cause e)+ (take-while some?)+ (some (fn [ex]+ (when (= error-type (:error (ex-data ex)))+ ex)))))+(defn source-error-message"Constructs user-friendly error messages from remote sync source exceptions.Takes a throwable exception and returns a string message that categorizes the error (network, authentication,repository not found, branch, or generic) based on the exception type and message content."[e]- (cond- (or (instance? java.net.UnknownHostException e)- (instance? java.net.UnknownHostException (ex-cause e)))- "Network error: Unable to reach git repository host"-- (str/includes? (ex-message e) "Authentication failed")- "Authentication failed: Please check your git credentials"-- (str/includes? (ex-message e) "Repository not found")- "Repository not found: Please check the repository URL"-- (str/includes? (ex-message e) "branch")- "Branch error: Please check the specified branch exists"-- (some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))- "This repository contains tenant collections, but the tenants feature is disabled on your instance."-- (str/includes? (ex-message e) "Missing commit")- "Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."-- (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)- (let [{:keys [model id]} (ex-data e)]- (format "Import failed: %s '%s' does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing." model id))-- (some-> e ex-cause ex-message (str/includes? "database not found"))- (format "Import failed: A referenced database does not exist on this instance. %s" (ex-message (ex-cause e)))-- (seq (:ingest-errors (ex-data e)))- (let [ingest-errors (:ingest-errors (ex-data e))]- (format "Failed to read %d file(s) from the repository: %s"- (count ingest-errors)- (str/join "; " (for [ie ingest-errors- :let [{:keys [file reason]} (ex-data ie)]]- (cond-> file- reason (str ": " reason))))))+ (let [missing-db (cause-with-error e :metabase.models.serialization.resolve.db/database-not-found)]+ (cond+ (or (instance? java.net.UnknownHostException e)+ (instance? java.net.UnknownHostException (ex-cause e)))+ "Network error: Unable to reach git repository host"++ (str/includes? (ex-message e) "Authentication failed")+ "Authentication failed: Please check your git credentials"++ (str/includes? (ex-message e) "Repository not found")+ "Repository not found: Please check the repository URL"++ (str/includes? (ex-message e) "branch")+ "Branch error: Please check the specified branch exists"++ (some-> e ex-cause ex-message (str/includes? "Can't create a tenant collection without tenants enabled"))+ "This repository contains tenant collections, but the tenants feature is disabled on your instance."++ (str/includes? (ex-message e) "Missing commit")+ "Repository cache is stale: the remote repository may have been force-pushed. Please retry the operation."++ (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/not-found)+ (let [{:keys [model id referrer]} (ex-data e)]+ (missing-reference-message {:missing {:model model :id id}+ :referrer referrer}))++ ;; the entity that failed to load is the one holding the reference to the absent database+ missing-db+ (missing-reference-message {:missing {:model "Database" :id (:db-name (ex-data missing-db))}+ :referrer (:entity (ex-data e))})++ (= (:error (ex-data e)) :metabase-enterprise.serialization.v2.load/load-failure)+ (let [{:keys [entity stripped-keys]} (ex-data e)]+ (load-failure-message {:entity entity+ :reason (some-> e ex-cause ex-message)+ :stripped-keys stripped-keys}))++ (seq (:ingest-errors (ex-data e)))+ (let [ingest-errors (:ingest-errors (ex-data e))]+ (format "Failed to read %d file(s) from the repository: %s"+ (count ingest-errors)+ (str/join "; " (for [ie ingest-errors+ :let [{:keys [file reason]} (ex-data ie)]]+ (cond-> (quoted file)+ reason (str ": " reason))))))- :else- (format "Failed to reload from git repository: %s" (ex-message e))))+ :else+ (format "Failed to reload from git repository: %s" (ex-message e)))))(defn- get-conflicts"Detects conflicts that would prevent or complicate import. Returns a map with two classes:diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex d9cad571a2e..1045ce00925 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -100,6 +100,30 @@:table (some->> last-model (keyword "model") t2/table-name):error error-type}))+(defn- entity-reference+ "Describes the entity at `path` as `{:model :id :name}` for use in user-facing error messages.+ `:name` is nil for models whose name is absent or not a plain string (e.g. Settings, templated Card names)."+ [path ingested]+ (let [self (last path)]+ {:model (:model self)+ :id (:id self)+ :name (when (string? (:name ingested))+ (:name ingested))}))++(defn- rethrow-with-referrer+ "Always throws. Rethrows `e` with `referrer` attached when it is a `::not-found` error that does not name one yet,+ and unchanged otherwise.++ Only the nearest enclosing [[load-one!]] attaches itself, since it unwinds first and later frames see the key+ already present. The replacement inherits `e`'s cause rather than nesting `e` beneath itself: error reporting+ renders the whole cause chain, so nesting would print the same message twice joined by `caused by:`."+ [e referrer]+ (let [data (ex-data e)]+ (if (and (= ::not-found (:error data))+ (not (contains? data :referrer)))+ (throw (ex-info (ex-message e) (assoc data :referrer referrer) (ex-cause e)))+ (throw e))))+(defn- valid-model-name-for-load? [model-name];; linear scan, but small n(->> (concat serdes.models/inlined-models@@ -179,11 +203,14 @@{:entity_id (:entity_id ingested):level (count expanding):deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))- ctx (-> ctx- (update :expanding conj path)- (load-deps! deps)- (update :seen conj path)- (update :expanding disj path))+ ctx (try+ (-> ctx+ (update :expanding conj path)+ (load-deps! deps)+ (update :seen conj path)+ (update :expanding disj path))+ (catch Exception e+ (rethrow-with-referrer e (entity-reference rebuilt-path ingested))))_ (when (seq deps)(log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested):level (count expanding)}))@@ -205,8 +232,9 @@(format " (it may have been left without these keys, which were stripped to break a circular dependency: %s)"(str/join ", " (sort (map name (keys-to-strip ingested)))))""))- (cond-> (path-error-data ::load-failure expanding path)- stripped? (assoc :stripped-keys (keys-to-strip ingested)))+ (-> (path-error-data ::load-failure expanding path)+ (assoc :entity (entity-reference rebuilt-path ingested))+ (cond-> stripped? (assoc :stripped-keys (keys-to-strip ingested))))e))))))))))(defn new-contextdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex fc15d5a598a..8fee1791581 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -104,15 +104,104 @@:model "Database":id "clickhouse":error :metabase-enterprise.serialization.v2.load/not-found})]- (is (= "Import failed: Database 'clickhouse' does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."+ (is (= "Import failed: Database (`clickhouse`) does not exist on this instance. Make sure all referenced databases and other dependencies are set up before importing."(impl/source-error-message e)))))- (testing "source-error-message produces helpful message for FK database-not-found errors"+ (testing "source-error-message names the entity holding the dangling reference when known (GHY-3992)"+ (let [e (ex-info "Collection 'xyz789' was not found"+ {:path "Collection xyz789"+ :model "Collection"+ :id "xyz789"+ :referrer {:model "Card" :id "abc123" :name "Orders by Month"}+ :error :metabase-enterprise.serialization.v2.load/not-found})]+ (is (= (str "Import failed: Card `Orders by Month` (`abc123`) references Collection (`xyz789`), which does not "+ "exist on this instance. Make sure all referenced databases and other dependencies are set up "+ "before importing.")+ (impl/source-error-message e)))))+ (testing "surrounding whitespace in a referenced name is visible because names are backtick-quoted (GHY-3992)"+ (doseq [db-name ["My Database " " My Database" "My Database"]]+ (let [e (ex-info (format "Database '%s' was not found" db-name)+ {:path (str "Database " db-name)+ :model "Database"+ :id db-name+ :error :metabase-enterprise.serialization.v2.load/not-found})]+ (is (str/includes? (impl/source-error-message e)+ (format "Database (`%s`)" db-name))+ (format "expected %s to be quoted verbatim" (pr-str db-name)))))))++(deftest source-error-message-database-not-found-test+ (testing "source-error-message names the card and the missing database for FK database-not-found errors"(let [cause (ex-info "table id present, but database not found: [clickhouse nil some_table]"- {:table-id ["clickhouse" nil "some_table"]})+ {:table-id ["clickhouse" nil "some_table"]+ :db-name "clickhouse"+ :error :metabase.models.serialization.resolve.db/database-not-found})+ e (ex-info "Failed to load into database for Card abc123"+ {:path "Card abc123"+ :entity {:model "Card" :id "abc123" :name "Some card"}}+ cause)]+ (is (= (str "Import failed: Card `Some card` (`abc123`) references Database (`clickhouse`), which does not "+ "exist on this instance. Make sure all referenced databases and other dependencies are set up "+ "before importing.")+ (impl/source-error-message e)))))+ (testing "database-not-found is found anywhere in the cause chain, not only at the immediate cause"+ (let [root (ex-info "table id present, but database not found: [clickhouse nil t]"+ {:db-name "clickhouse"+ :error :metabase.models.serialization.resolve.db/database-not-found})+ middle (ex-info "wrapped by an intervening helper" {} root)+ e (ex-info "Failed to load into database for Card abc123" {:path "Card abc123"} middle)]+ (is (str/includes? (impl/source-error-message e) "Database (`clickhouse`)")))))++(deftest source-error-message-load-failure-test+ (testing "source-error-message names the entity and the underlying reason (GHY-3992)"+ (let [cause (ex-info "NOT NULL constraint failed: report_card.display" {})e (ex-info "Failed to load into database for Card abc123"- {:path "Card abc123"}+ {:path "Card abc123"+ :entity {:model "Card" :id "abc123" :name "Orders by Month"}+ :error :metabase-enterprise.serialization.v2.load/load-failure}+ cause)]+ (is (= (str "Import failed: could not save Card `Orders by Month` (`abc123`). "+ "NOT NULL constraint failed: report_card.display.")+ (impl/source-error-message e)))))+ (testing "a reason that already ends in a period is not double-punctuated"+ (let [e (ex-info "Failed to load into database for Card abc123"+ {:entity {:model "Card" :id "abc123" :name "Orders by Month"}+ :error :metabase-enterprise.serialization.v2.load/load-failure}+ (ex-info "Something went wrong." {}))]+ (is (str/ends-with? (impl/source-error-message e) "went wrong."))))+ (testing "a load-failure with no cause omits the reason clause"+ (let [e (ex-info "Failed to load into database for Card abc123"+ {:entity {:model "Card" :id "abc123" :name "Orders by Month"}+ :error :metabase-enterprise.serialization.v2.load/load-failure})]+ (is (= "Import failed: could not save Card `Orders by Month` (`abc123`)."+ (impl/source-error-message e)))))+ (testing "stripped keys are reported, since a partial row may have been committed (GHY-3992)"+ (let [cause (ex-info "some db error" {})+ e (ex-info "Failed to load into database for Dashboard xyz"+ {:path "Dashboard xyz"+ :entity {:model "Dashboard" :id "xyz" :name "Sales"}+ :stripped-keys #{:parameters :dashcards}+ :error :metabase-enterprise.serialization.v2.load/load-failure}cause)]- (is (str/includes? (impl/source-error-message e) "A referenced database does not exist on this instance"))))+ (is (= (str "Import failed: could not save Dashboard `Sales` (`xyz`). some db error. "+ "It may have been saved without: `dashcards`, `parameters`.")+ (impl/source-error-message e)))))+ (testing "a database-not-found cause still wins over the generic load-failure branch"+ (let [cause (ex-info "table id present, but database not found: [ch nil t]"+ {:db-name "ch"+ :error :metabase.models.serialization.resolve.db/database-not-found})+ e (ex-info "Failed to load into database for Card abc123"+ {:entity {:model "Card" :id "abc123" :name "Some card"}+ :error :metabase-enterprise.serialization.v2.load/load-failure}+ cause)]+ (is (str/includes? (impl/source-error-message e) "references Database (`ch`)"))))+ (testing "a tenant-collection cause still wins over the generic load-failure branch"+ (let [cause (ex-info "Can't create a tenant collection without tenants enabled" {})+ e (ex-info "Failed to load into database for Collection abc"+ {:entity {:model "Collection" :id "abc"}+ :error :metabase-enterprise.serialization.v2.load/load-failure}+ cause)]+ (is (str/includes? (impl/source-error-message e) "tenants feature is disabled")))))++(deftest source-error-message-ingest-errors-test(testing "source-error-message lists each unreadable file with its parse reason (GHY-3887)"(let [ingest-err (ex-info "Failed to parse file: collections/transforms/a.yaml"{:file "collections/transforms/a.yaml"@@ -123,8 +212,23 @@ingest-err)msg (impl/source-error-message e)](is (str/includes? msg "Failed to read 1 file(s)"))- (is (str/includes? msg "collections/transforms/a.yaml"))- (is (str/includes? msg "found character '@'")))))+ (is (str/includes? msg "`collections/transforms/a.yaml`"))+ (is (str/includes? msg "found character '@'"))))+ (testing "file paths are backtick-quoted so surrounding whitespace is visible (GHY-3992)"+ (let [ingest-err (ex-info "Failed to read file: collections/bar .yaml"+ {:file "collections/bar .yaml"+ :reason "IOException"})+ e (ex-info "Failed to read 1 file(s) during ingestion: collections/bar .yaml"+ {:ingest-errors [ingest-err]}+ ingest-err)]+ (is (= "Failed to read 1 file(s) from the repository: `collections/bar .yaml`: IOException"+ (impl/source-error-message e)))))+ (testing "each unreadable file is quoted independently when several fail"+ (let [errs [(ex-info "a" {:file "a.yaml" :reason "bad"})+ (ex-info "b" {:file "b.yaml" :reason "worse"})]+ e (ex-info "Failed to read 2 file(s) during ingestion" {:ingest-errors errs} (first errs))]+ (is (= "Failed to read 2 file(s) from the repository: `a.yaml`: bad; `b.yaml`: worse"+ (impl/source-error-message e))))));; We need to make sure the task-id we use to track the Remote Sync is not bound to a transactions because of the behavior of;; update-sync-progress. So the follow two tests cannot use with-temp to create modelsdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex d2d5b122048..e64a8bb9e96 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -1246,10 +1246,58 @@:entity_id "0123456789abcdef_0123":name "Some card":table_id ["bad-db" nil "CUSTOMERS"]- :visualization_settings {}}])]- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"was not found"- (serdes.load/load-metabase! ingestion)))))))))+ :visualization_settings {}}])+ e (try+ (serdes.load/load-metabase! ingestion)+ nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? e))+ (is (re-find #"was not found" (ex-message e)))+ (testing "the not-found error names the entity holding the dangling reference (GHY-3992)"+ (is (= {:model "Database" :id "bad-db"}+ (select-keys (ex-data e) [:model :id])))+ (is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}+ (:referrer (ex-data e)))))+ ;; attaching the referrer must not nest the original beneath itself: error reporting renders the+ ;; whole cause chain, and a self-nested exception prints the same message twice+ (testing "attaching the referrer does not duplicate the error through the cause chain"+ (is (nil? (ex-cause e))))))+ ;; `result-metadata-deps` derives deps only from each entry's `:field_ref`, never from its `:table_id`.+ ;; So a result_metadata `:table_id` naming an absent database gets past dependency resolution and only+ ;; fails inside `serdes/load-one!`, where `import-table-fk` raises ::database-not-found. (The Card's own+ ;; top-level `table_id` cannot be used here: `populate-query-fields` recomputes it from `dataset_query`.)+ (testing "a result_metadata table_id naming an absent database names the database and the card (GHY-3992)"+ (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]+ :created_at (t/instant)+ :creator_id "[email protected]"+ :database_id "my-db"+ :dataset_query {:database "my-db"+ :type :query+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}+ :display :table+ :entity_id "0123456789abcdef_0123"+ :name "Some card"+ :result_metadata [{:name "STATE"+ :base_type :type/Text+ :table_id ["absent-db" nil "CUSTOMERS"]}]+ :table_id ["my-db" nil "CUSTOMERS"]+ :visualization_settings {}}])+ e (try+ (serdes.load/load-metabase! ingestion)+ nil+ (catch clojure.lang.ExceptionInfo e e))+ root (->> (iterate ex-cause e)+ (take-while some?)+ (some #(when (= :metabase.models.serialization.resolve.db/database-not-found+ (:error (ex-data %)))+ %)))]+ (is (some? e))+ (testing "the root cause carries the missing database name"+ (is (some? root))+ (is (= "absent-db" (:db-name (ex-data root)))))+ (testing "the outer load-failure names the card that failed to load"+ (is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}+ (:entity (ex-data e)))))))))))(deftest card-with-snippet-test(let [db1s (atom nil)diff --git a/src/metabase/models/serialization/resolve/db.clj b/src/metabase/models/serialization/resolve/db.cljindex d8ab17369c8..813a85c66d4 100644--- a/src/metabase/models/serialization/resolve/db.clj+++ b/src/metabase/models/serialization/resolve/db.clj@@ -126,8 +126,10 @@(or (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id db-id)(synthesize-table! db-id schema table-name))(throw (ex-info (format "table id present, but database not found: %s" table-id)- {:table-id table-id- :database-names (sort (t2/select-fn-vec :name :model/Table))})))))+ {:table-id table-id+ :db-name db-name+ :database-names (sort (t2/select-fn-vec :name :model/Database))+ :error ::database-not-found})))))(defn import-field-fk"Given [db-name schema table-name field-name ...], return numeric field_id. If part of the parent