The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[serdes] do not say "Loading ..." if entity is not being loaded (#64461)
Oleksandr Solovyovaf0b621eeb33
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.cljindex 860ea00bbe1..7d1aa2edb13 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj@@ -85,20 +85,20 @@(concat (for [k (keys settings)][{:model "Setting" :id (name k)}]))))- (ingest-one [_ abs-path]+ (ingest-one [_ serdes-meta](when-not @cache(reset! cache (ingest-all root-dir)))- (let [{:keys [id]} (first abs-path)+ (let [{:keys [id]} (first serdes-meta)kw-id (keyword id)]- (if (= ["Setting"] (mapv :model abs-path))- {:serdes/meta abs-path :key kw-id :value (get settings kw-id)}- (if-let [target (get @cache (strip-labels abs-path))]+ (if (= ["Setting"] (mapv :model serdes-meta))+ (when (contains? settings kw-id)+ {:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})+ (when-let [target (get @cache (strip-labels serdes-meta))](try(ingest-file (second target))(catch Exception e(throw (ex-info "Unable to ingest file" {:file (.getName ^File (second target))- :abs-path abs-path} e))))- (throw (ex-info "Cannot find file" {:abs-path abs-path})))))))+ :abs-path serdes-meta} e)))))))))(defn ingest-yaml"Creates a new Ingestable on a directory of YAML files, as created bydiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 2752e72cc50..2cd1c5a7185 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -94,8 +94,8 @@The only tangling stuff is handling circular dependencies: parts of this is handled by the `serdes/load-one!`function, just outright skipping processing for parts of ingested data."[{:keys [expanding seen circular ingestion] :as ctx} path]- (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}- (circular path) (assoc :stripped true)))+ (log/debug "Requested" (cond-> {:path (serdes/log-path-str path)}+ (circular path) (assoc :stripped true)))(cond(and (expanding path)(circular path)) (throw (ex-info (format "Circular dependency on %s" (serdes/log-path-str path))@@ -108,51 +108,54 @@path))(seen path) ctx ; Already been done, can skip it.:else- (let [ingested (try- (serdes.ingest/ingest-one ingestion path)- (catch Exception e- (throw (ex-info (format "Failed to read file for %s" (serdes/log-path-str path))- (path-error-data ::not-found expanding path)- e))))- ;; Use the abstract path as attached by the ingestion process, not the original one we were passed.- rebuilt-path (serdes/path ingested)- ;; If nil or absent :entity_id is taken as a signal to create a new entity- ;; To get a nil entity_id, a user has to manually set the entity_id to null or remove it- ;; in the yaml file.- ;; In all other cases we should expect an :entity_id:- ;; - exported entities have a :entity_id for every model that can have one- ;; - backfill (pre import) guarantees all entities have ids in the appdb- expect-entity-id (some-> rebuilt-path peek :model exported-with-entity-id?)- require-new-entity (and expect-entity-id (nil? (:entity_id ingested)))- ingested (cond-> ingested- require-new-entity (assoc :entity_id (u/generate-nano-id))- ;; `::strip` is handled in `metabase.serialization`- (circular path) (assoc ::serdes/strip (keys-to-strip ingested)))- ;; we need less deps when trying to load "stripped" data- deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))- (remove seen))- _ (when (seq deps)- (log/debug "Loading dependencies"- {:entity_id (:entity_id ingested)- :level (count expanding)- :deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))- ctx (-> ctx- (update :expanding conj path)- (load-deps! deps)- (update :seen conj path)- (update :expanding disj path))- _ (when (seq deps)- (log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested)- :level (count expanding)}))- local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]- (try- (serdes/load-one! ingested local-or-nil)- ctx- (catch Exception e- ;; ugly mapv here to convered #ordered/map into normal map so it's readable in the logs- (throw (ex-info (format "Failed to load into database for %s" (serdes/log-path-str path))- (path-error-data ::load-failure expanding path)- e)))))))+ (let [ingested (serdes.ingest/ingest-one ingestion path)]+ (if-not ingested+ (do+ (when-not (serdes/load-find-local path)+ (throw (ex-info "Failed to read file" {:path (serdes/log-path-str path)})))+ (log/debug "Local" {:path (serdes/log-path-str path)})+ ctx)+ (let [_ (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}+ (circular path) (assoc :stripped true)))+ ;; Use the abstract path as attached by the ingestion process, not the original one we were passed.+ rebuilt-path (serdes/path ingested)+ ;; If nil or absent :entity_id is taken as a signal to create a new entity+ ;; To get a nil entity_id, a user has to manually set the entity_id to null or remove it+ ;; in the yaml file.+ ;; In all other cases we should expect an :entity_id:+ ;; - exported entities have a :entity_id for every model that can have one+ ;; - backfill (pre import) guarantees all entities have ids in the appdb+ expect-entity-id (some-> rebuilt-path peek :model exported-with-entity-id?)+ require-new-entity (and expect-entity-id (nil? (:entity_id ingested)))+ ingested (cond-> ingested+ require-new-entity (assoc :entity_id (u/generate-nano-id))+ ;; `::strip` is handled in `metabase.serialization`+ (circular path) (assoc ::serdes/strip (keys-to-strip ingested)))+ ;; we need less deps when trying to load "stripped" data+ deps (->> (serdes/dependencies (apply dissoc ingested (::serdes/strip ingested)))+ (remove seen))+ _ (when (seq deps)+ (log/debug "Loading dependencies"+ {:entity_id (:entity_id ingested)+ :level (count expanding)+ :deps (str "[" (str/join ", " (map serdes/log-path-str deps)) "]")}))+ ctx (-> ctx+ (update :expanding conj path)+ (load-deps! deps)+ (update :seen conj path)+ (update :expanding disj path))+ _ (when (seq deps)+ (log/debug "Ended loading dependencies" {:entity_id (:entity_id ingested)+ :level (count expanding)}))+ local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]+ (try+ (serdes/load-one! ingested local-or-nil)+ ctx+ (catch Exception e+ ;; ugly mapv here to convered #ordered/map into normal map so it's readable in the logs+ (throw (ex-info (format "Failed to load into database for %s" (serdes/log-path-str path))+ (path-error-data ::load-failure expanding path)+ e)))))))))(defn load-metabase!"Loads in a database export from an ingestion source, which is any Ingestable instance."diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.cljindex d79c31dba64..90bc0a380ba 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj@@ -47,7 +47,8 @@[stream root-dir](let [settings (atom [])report (atom {:seen [] :errors []})- opts (merge {:root-dir root-dir} (serdes/storage-base-context))]+ opts (-> (serdes/storage-base-context)+ (assoc :root-dir root-dir))](doseq [entity stream](cond(instance? Exception entity)diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex ba4f0529148..f324d636723 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -239,7 +239,7 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file ba})](testing "We get our data items back"- (is (= #{"Collection" "Dashboard" "Card" "Database" "TransformTag" "TransformJob"}+ (is (= #{"Collection" "Dashboard" "Card" "TransformTag" "TransformJob"}(log-types (line-seq (io/reader (io/input-stream res)))))))(testing "And they hit the db"(is (= (:name dash) (t2/select-one-fn :name :model/Dashboard :entity_id (:entity_id dash))))@@ -272,16 +272,7 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file ba}))log (slurp (io/input-stream res))]- (testing "3 header lines, then cards+database+collection, then the error"- (is (re-find #"Failed to read file for Collection DoesNotExist" log))- (is (re-find #"Cannot find file" log)) ;; underlying error- (is (= {:deps-chain #{[{:id "**ID**", :model "Card"}]},- :error :metabase-enterprise.serialization.v2.load/not-found,- :model "Collection",- :path [{:id "DoesNotExist", :model "Collection"}],- :local-id nil- :table :collection}- (extract-and-sanitize-exception-map log))))+ (is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log))(testing "Snowplow event about error was sent"(is (=? {"success" false"event" "serialization"@@ -290,7 +281,7 @@"duration_ms" int?"count" 0"error_count" 0- "error_message" #"(?s)Failed to read file for Collection DoesNotExist.*"}+ "error_message" "Failed to read file {:path \"Collection DoesNotExist\"}"}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))(testing "Skipping errors /api/ee/serialization/import"@@ -300,9 +291,9 @@:continue_on_error true)log (slurp (io/input-stream res))](testing "3 header lines, then card+database+coll, error, then dashboard+coll"- (is (= #{"Dashboard" "Card" "Database" "Collection" "TransformTag" "TransformJob"}+ (is (= #{"Dashboard" "Card" "Collection" "TransformTag" "TransformJob"}(log-types (str/split-lines log))))- (is (re-find #"Failed to read file for Collection DoesNotExist" log)))+ (is (re-find #"Failed to read file \{:path \"Collection DoesNotExist\"}" log)))(testing "Snowplow event about error was sent"(is (=? {"success" true"event" "serialization"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.cljindex c9b31fd4e5a..f5ce12f0f1a 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj@@ -251,7 +251,7 @@"count" 0"success" false;; t2/with-transactions re-wraps errors with data about toucan connections- "error_message" #"(?s)Failed to read file for Collection DoesNotExist.*"}+ "error_message" "Failed to read file {:path \"Collection DoesNotExist\"}"}(-> (snowplow-test/pop-event-data-and-user-id!) first :data)))))))))))))(deftest entity-id-dump&load-testdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex 24154306cfd..cb7aa80a524 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -42,9 +42,7 @@(ingest-list [_](keys mapped))(ingest-one [_ path]- (or (get mapped (no-labels path))- (throw (ex-info (format "Unknown ingestion target: %s" path)- {:path path :world mapped})))))))+ (get mapped (no-labels path))))));;; WARNING for test authors: [[extract/extract]] returns a lazy reducible value. To make sure you don't;;; confound your tests with data from your dev appdb, remember to eagerly@@ -1324,8 +1322,8 @@(let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})](is (= 1 (count (:errors report))))(is (= 3 (count (:seen report)))))- (is (= [["Failed to read file for Collection does-not-exist"]]- (logs-extract #"Skipping deserialization error: (.*) \{.*\}\n"+ (is (= [["Failed to read file {:path \"Collection does-not-exist\"}"]]+ (logs-extract #"Skipping deserialization error: (.*)"(messages)))))))))))(deftest with-dbs-works-as-expected-test