Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stale appdb index drop improvements (#72910)

Nathan Voxland180bf74cda45
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index e60f63b8f3e..7c413d6c45d 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -2,7 +2,6 @@
(:require
[clojure.set :as set]
[clojure.string :as str]
- [honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
[metabase.analytics.core :as analytics]
[metabase.app-db.core :as mdb]
@@ -23,7 +22,6 @@
[metabase.util.string :as string]
[toucan2.core :as t2])
(:import
- (clojure.lang ExceptionInfo)
(org.h2.jdbc JdbcSQLSyntaxErrorException)
(org.postgresql.util PSQLException)))
@@ -129,12 +127,9 @@
;; Exclude temp tables — they are managed by with-temp-index-table
[:not-like [:lower :table_name] [:inline "%\\_temp"]]
[:not-in [:lower :table_name]
- [:raw
- (str "("
- (first (sql/format {:select [:index_name]
- :from [[(t2/table-name :model/SearchIndexMetadata) :metadata]]
- :where [:= :metadata.engine [:inline "appdb"]]}))
- ")")]]]})))
+ {:select [:%lower.index_name]
+ :from [(t2/table-name :model/SearchIndexMetadata)]
+ :where [:= :engine [:inline "appdb"]]}]]})))
(defn- delete-obsolete-tables! []
;; Delete metadata around indexes that are no longer needed.
@@ -146,7 +141,8 @@
(t2/query (sql.helpers/drop-table table))
(vswap! dropped conj table)
;; Deletion could fail if it races with other instances
- (catch ExceptionInfo _)))
+ (catch Exception e
+ (log/warnf e "Failed to drop stale index %s" table))))
(log/infof "Dropped %d stale indexes: %s" (count @dropped) @dropped)))
(defn- ->db-type [t]
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 4bad3a82392..6164d6afb75 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -205,12 +205,34 @@
(-> (disj "indexed-entity") (concat ["indexed-entity"])))]
(reduce u/rconcat [] (map spec-index-reducible sorted-models))))
+(def ^:private max-document-error-logs 10)
+
(defn- query->documents [query-reducible]
- (->> query-reducible
- (eduction
- (comp
- (map t2.realize/realize)
- (map ->document)))))
+ (let [failures (volatile! 0)]
+ (->> query-reducible
+ (eduction
+ (comp
+ (map t2.realize/realize)
+ (fn [rf]
+ (fn
+ ([] (rf))
+ ([result]
+ (when (pos? @failures)
+ (log/warnf "Failed to build %d search documents; they were skipped" @failures))
+ (rf result))
+ ([result m]
+ (if-let [doc (try
+ (->document m)
+ (catch Throwable t
+ (let [n (vswap! failures inc)]
+ (cond
+ (<= n max-document-error-logs)
+ (log/warnf t "Error building search document for %s %s; skipping" (:model m) (:id m))
+ (= n (inc max-document-error-logs))
+ (log/warnf "Suppressing further per-document error logs after %d failures" max-document-error-logs)))
+ nil))]
+ (rf result doc)
+ result)))))))))
(defn searchable-documents
"Return all existing searchable documents from the database."