Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add flag to skip search reindexing on import (#65537)

Chris Truter5937d3bb38f6
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 916b758df1c..f7df1ae2ed1 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -116,10 +116,12 @@
(defn- unpack&import [^File file & [{:keys [size
continue-on-error
- full-stacktrace]}]]
+ full-stacktrace
+ reindex?]}]]
(let [dst (io/file parent-dir (u.random/random-name))
log-file (io/file dst "import.log")
err (atom nil)
+ reindex? (if (nil? reindex?) true reindex?)
report (with-open [_logger (logger/for-ns log-file ['metabase-enterprise.serialization
'metabase.models.serialization]
{:additive *additive-logging*})]
@@ -138,7 +140,8 @@
(log/infof "In total %s entries unpacked, detected source dir: %s" cnt (.getName path))
(serdes/with-cache
(-> (v2.ingest/ingest-yaml (.getPath path))
- (v2.load/load-metabase! {:continue-on-error continue-on-error}))))
+ (v2.load/load-metabase! {:continue-on-error continue-on-error
+ :reindex? reindex?}))))
(catch Exception e
(reset! err e)
(if full-stacktrace
@@ -249,10 +252,15 @@
[_route-params
{continue-on-error? :continue_on_error
full-stacktrace? :full_stacktrace
+ reindex-search? :reindex
:as _query-params}
:- [:map
[:continue_on_error {:default false} (mu/with ms/BooleanValue {:description "Do not break execution on errors"})]
- [:full_stacktrace {:default false} (mu/with ms/BooleanValue {:description "Show full stacktraces in the logs"})]]
+ [:full_stacktrace {:default false} (mu/with ms/BooleanValue {:description "Show full stacktraces in the logs"})]
+ ;; TODO this parameter is a kludge to fix https://linear.app/metabase/issue/GDGT-573
+ ;; ideally we'd fix the underlying issue (by delaying realtime indexing updates until the tx closes)
+ ;; for now, we let users opt out, in case they're indexing a lot, so they can only reindex on the last step
+ [:reindex {:default true} (mu/with ms/BooleanValue {:description "Rebuild the search index afterwards"})]]
_body
{{:strs [file]} :multipart-params, :as _request} :- [:map
[:multipart-params
@@ -268,7 +276,8 @@
callback]} (unpack&import (:tempfile file)
{:size (:size file)
:continue-on-error continue-on-error?
- :full-stacktrace full-stacktrace?})
+ :full-stacktrace full-stacktrace?
+ :reindex? reindex-search?})
imported (into (sorted-set) (map (comp :model last)) (:seen report))]
(analytics/track-event! :snowplow/serialization
{:event :serialization
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index f324d636723..698106f9e7c 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -185,7 +185,11 @@
(testing "POST /api/ee/serialization/export"
(mt/with-temp [:model/Collection coll {}
:model/Dashboard dash {:collection_id (:id coll), :name "thraddash"}
- :model/Card card {:collection_id (:id coll), :name "frobinate", :type :model}]
+ :model/Card card {:collection_id (:id coll), :name "frobinate", :type :model
+ :query_type :native
+ :dataset_query {:type :native
+ :database (t2/select-one-pk :model/Database)
+ :native {:query "SELECT 1"}}}]
(testing "We clear the card from the search index"
(is (= 1 (search-result-count "dashboard" "thraddash")))
@@ -235,9 +239,11 @@
(t2/update! :model/Dashboard {:id (:id dash)} {:name "urquan"})
(t2/delete! :model/Card (:id card))
- (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/import"
- {:request-options {:headers {"content-type" "multipart/form-data"}}}
- {:file ba})]
+ (let [re-indexed? (atom false)
+ res (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
+ (mt/user-http-request :crowberto :post 200 "ee/serialization/import?reindex=false"
+ {:request-options {:headers {"content-type" "multipart/form-data"}}}
+ {:file ba}))]
(testing "We get our data items back"
(is (= #{"Collection" "Dashboard" "Card" "TransformTag" "TransformJob"}
(log-types (line-seq (io/reader (io/input-stream res)))))))
@@ -254,12 +260,15 @@
"error_count" 0
"success" true
"error_message" nil}
- (-> (snowplow-test/pop-event-data-and-user-id!) last :data)))))
+ (-> (snowplow-test/pop-event-data-and-user-id!) last :data))))
- (testing "The loaded entities are added to the search index"
- (is (= 1 (search-result-count "dashboard" "thraddash")))
- (is (= 0 (search-result-count "dashboard" "urquan")))
- (is (= 1 (search-result-count "dataset" "frobinate")))))
+ (testing "we did not re-index"
+ (is (false? @re-indexed?)))
+
+ (testing "The loaded entities are added to the search index"
+ (is (= 1 (search-result-count "dashboard" "thraddash")))
+ (is (= 0 (search-result-count "dashboard" "urquan")))
+ (is (= 1 (search-result-count "dataset" "frobinate"))))))
(mt/with-dynamic-fn-redefs [v2.ingest/ingest-file (let [ingest-file (mt/dynamic-value #'v2.ingest/ingest-file)]
(fn [^File file]