Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[serdes] do not say "Loading ..." if entity is not being loaded (#64461)

Oleksandr Solovyovaf0b621eeb33
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index 860ea00bbe1..7d1aa2edb13 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -85,20 +85,20 @@
(concat (for [k (keys settings)]
[{:model "Setting" :id (name k)}]))))
- (ingest-one [_ abs-path]
+ (ingest-one [_ serdes-meta]
(when-not @cache
(reset! cache (ingest-all root-dir)))
- (let [{:keys [id]} (first abs-path)
+ (let [{:keys [id]} (first serdes-meta)
kw-id (keyword id)]
- (if (= ["Setting"] (mapv :model abs-path))
- {:serdes/meta abs-path :key kw-id :value (get settings kw-id)}
- (if-let [target (get @cache (strip-labels abs-path))]
+ (if (= ["Setting"] (mapv :model serdes-meta))
+ (when (contains? settings kw-id)
+ {:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})
+ (when-let [target (get @cache (strip-labels serdes-meta))]
(try
(ingest-file (second target))
(catch Exception e
(throw (ex-info "Unable to ingest file" {:file (.getName ^File (second target))
- :abs-path abs-path} e))))
- (throw (ex-info "Cannot find file" {:abs-path abs-path})))))))
+ :abs-path serdes-meta} e)))))))))
(defn ingest-yaml
"Creates a new Ingestable on a directory of YAML files, as created by
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 2752e72cc50..2cd1c5a7185 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -94,8 +94,8 @@
The only tangling stuff is handling circular dependencies: parts of this is handled by the `serdes/load-one!`
function, just outright skipping processing for parts of ingested data."
[{:keys [expanding seen circular ingestion] :as ctx} path]
- (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}
- (circular path) (assoc :stripped true)))
+ (log/debug "Requested" (cond-> {:path (serdes/log-path-str path)}
+ (circular path) (assoc :stripped true)))
(cond
(and (expanding path)
(circular path)) (throw (ex-info (format "Circular dependency on %s" (serdes/log-path-str path))
@@ -108,51 +108,54 @@
path))
(seen path) ctx ; Already been done, can skip it.
:else
- (let [ingested (try
- (serdes.ingest/ingest-one ingestion path)
- (catch Exception e
- (throw (ex-info (format "Failed to read file for %s" (serdes/log-path-str path))
- (path-error-data ::not-found expanding path)
- e))))
- ;; Use the abstract path as attached by the ingestion process, not the original one we were passed.
- rebuilt-path (serdes/path ingested)
- ;; If nil or absent :entity_id is taken as a signal to create a new entity
- ;; To get a nil entity_id, a user has to manually set the entity_id to null or remove it
- ;; in the yaml file.
- ;; In all other cases we should expect an :entity_id:
- ;; - exported entities have a :entity_id for every model that can have one
- ;; - backfill (pre import) guarantees all entities have ids in the appdb
- expect-entity-id (some-> rebuilt-path peek :model exported-with-entity-id?)
- require-new-entity (and expect-entity-id (nil? (:entity_id ingested)))
- ingested (cond-> ingested
- require-new-entity (assoc :entity_id (u/generate-nano-id))
- ;; `::strip` is handled in `metabase.serialization`
- (circular path) (assoc ::serdes/strip (keys-to-strip ingested)))
- ;; we need less deps when trying to load "stripped" data
- deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))
- (remove seen))
- _ (when (seq deps)
- (log/debug "Loading dependencies"
- {:entity_id (:entity_id ingested)
- :level (count expanding)
- :deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))
- ctx (-> ctx
- (update :expanding conj path)
- (load-deps! deps)
- (update :seen conj path)
- (update :expanding disj path))
- _ (when (seq deps)
- (log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested)
- :level (count expanding)}))
- local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]
- (try
- (serdes/load-one! ingested local-or-nil)
- ctx
- (catch Exception e
- ;; ugly mapv here to convered #ordered/map into normal map so it's readable in the logs
- (throw (ex-info (format "Failed to load into database for %s" (serdes/log-path-str path))
- (path-error-data ::load-failure expanding path)
- e)))))))
+ (let [ingested (serdes.ingest/ingest-one ingestion path)]
+ (if-not ingested
+ (do
+ (when-not (serdes/load-find-local path)
+ (throw (ex-info "Failed to read file" {:path (serdes/log-path-str path)})))
+ (log/debug "Local" {:path (serdes/log-path-str path)})
+ ctx)
+ (let [_ (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}
+ (circular path) (assoc :stripped true)))
+ ;; Use the abstract path as attached by the ingestion process, not the original one we were passed.
+ rebuilt-path (serdes/path ingested)
+ ;; If nil or absent :entity_id is taken as a signal to create a new entity
+ ;; To get a nil entity_id, a user has to manually set the entity_id to null or remove it
+ ;; in the yaml file.
+ ;; In all other cases we should expect an :entity_id:
+ ;; - exported entities have a :entity_id for every model that can have one
+ ;; - backfill (pre import) guarantees all entities have ids in the appdb
+ expect-entity-id (some-> rebuilt-path peek :model exported-with-entity-id?)
+ require-new-entity (and expect-entity-id (nil? (:entity_id ingested)))
+ ingested (cond-> ingested
+ require-new-entity (assoc :entity_id (u/generate-nano-id))
+ ;; `::strip` is handled in `metabase.serialization`
+ (circular path) (assoc ::serdes/strip (keys-to-strip ingested)))
+ ;; we need less deps when trying to load "stripped" data
+ deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))
+ (remove seen))
+ _ (when (seq deps)
+ (log/debug "Loading dependencies"
+ {:entity_id (:entity_id ingested)
+ :level (count expanding)
+ :deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))
+ ctx (-> ctx
+ (update :expanding conj path)
+ (load-deps! deps)
+ (update :seen conj path)
+ (update :expanding disj path))
+ _ (when (seq deps)
+ (log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested)
+ :level (count expanding)}))
+ local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]
+ (try
+ (serdes/load-one! ingested local-or-nil)
+ ctx
+ (catch Exception e
+ ;; ugly mapv here to convered #ordered/map into normal map so it's readable in the logs
+ (throw (ex-info (format "Failed to load into database for %s" (serdes/log-path-str path))
+ (path-error-data ::load-failure expanding path)
+ e)))))))))
(defn load-metabase!
"Loads in a database export from an ingestion source, which is any Ingestable instance."
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
index d79c31dba64..90bc0a380ba 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
@@ -47,7 +47,8 @@
[stream root-dir]
(let [settings (atom [])
report (atom {:seen [] :errors []})
- opts (merge {:root-dir root-dir} (serdes/storage-base-context))]
+ opts (-> (serdes/storage-base-context)
+ (assoc :root-dir root-dir))]
(doseq [entity stream]
(cond
(instance? Exception entity)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index ba4f0529148..f324d636723 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -239,7 +239,7 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba})]
(testing "We get our data items back"
- (is (= #{"Collection" "Dashboard" "Card" "Database" "TransformTag" "TransformJob"}
+ (is (= #{"Collection" "Dashboard" "Card" "TransformTag" "TransformJob"}
(log-types (line-seq (io/reader (io/input-stream res)))))))
(testing "And they hit the db"
(is (= (:name dash) (t2/select-one-fn :name :model/Dashboard :entity_id (:entity_id dash))))
@@ -272,16 +272,7 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba}))
log (slurp (io/input-stream res))]
- (testing "3 header lines, then cards+database+collection, then the error"
- (is (re-find #"Failed to read file for Collection DoesNotExist" log))
- (is (re-find #"Cannot find file" log)) ;; underlying error
- (is (= {:deps-chain #{[{:id "**ID**", :model "Card"}]},
- :error :metabase-enterprise.serialization.v2.load/not-found,
- :model "Collection",
- :path [{:id "DoesNotExist", :model "Collection"}],
- :local-id nil
- :table :collection}
- (extract-and-sanitize-exception-map log))))
+ (is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log))
(testing "Snowplow event about error was sent"
(is (=? {"success" false
"event" "serialization"
@@ -290,7 +281,7 @@
"duration_ms" int?
"count" 0
"error_count" 0
- "error_message" #"(?s)Failed to read file for Collection DoesNotExist.*"}
+ "error_message" "Failed to read file {:path \"Collection DoesNotExist\"}"}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))
(testing "Skipping errors /api/ee/serialization/import"
@@ -300,9 +291,9 @@
:continue_on_error true)
log (slurp (io/input-stream res))]
(testing "3 header lines, then card+database+coll, error, then dashboard+coll"
- (is (= #{"Dashboard" "Card" "Database" "Collection" "TransformTag" "TransformJob"}
+ (is (= #{"Dashboard" "Card" "Collection" "TransformTag" "TransformJob"}
(log-types (str/split-lines log))))
- (is (re-find #"Failed to read file for Collection DoesNotExist" log)))
+ (is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))
(testing "Snowplow event about error was sent"
(is (=? {"success" true
"event" "serialization"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
index c9b31fd4e5a..f5ce12f0f1a 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
@@ -251,7 +251,7 @@
"count" 0
"success" false
;; t2/with-transactions re-wraps errors with data about toucan connections
- "error_message" #"(?s)Failed to read file for Collection DoesNotExist.*"}
+ "error_message" "Failed to read file {:path \"Collection DoesNotExist\"}"}
(-> (snowplow-test/pop-event-data-and-user-id!) first :data)))))))))))))
(deftest entity-id-dump&load-test
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 24154306cfd..cb7aa80a524 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -42,9 +42,7 @@
(ingest-list [_]
(keys mapped))
(ingest-one [_ path]
- (or (get mapped (no-labels path))
- (throw (ex-info (format "Unknown ingestion target: %s" path)
- {:path path :world mapped})))))))
+ (get mapped (no-labels path))))))
;;; WARNING for test authors: [[extract/extract]] returns a lazy reducible value. To make sure you don't
;;; confound your tests with data from your dev appdb, remember to eagerly
@@ -1324,8 +1322,8 @@
(let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})]
(is (= 1 (count (:errors report))))
(is (= 3 (count (:seen report)))))
- (is (= [["Failed to read file for Collection does-not-exist"]]
- (logs-extract #"Skipping deserialization error: (.*) \{.*\}\n"
+ (is (= [["Failed to read file {:path \"Collection does-not-exist\"}"]]
+ (logs-extract #"Skipping deserialization error: (.*)"
(messages)))))))))))
(deftest with-dbs-works-as-expected-test