The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stale appdb index drop improvements (#72910)
Nathan Voxland180bf74cda45
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex e60f63b8f3e..7c413d6c45d 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -2,7 +2,6 @@(:require[clojure.set :as set][clojure.string :as str]- [honey.sql :as sql][honey.sql.helpers :as sql.helpers][metabase.analytics.core :as analytics][metabase.app-db.core :as mdb]@@ -23,7 +22,6 @@[metabase.util.string :as string][toucan2.core :as t2])(:import- (clojure.lang ExceptionInfo)(org.h2.jdbc JdbcSQLSyntaxErrorException)(org.postgresql.util PSQLException)))@@ -129,12 +127,9 @@;; Exclude temp tables — they are managed by with-temp-index-table[:not-like [:lower :table_name] [:inline "%\\_temp"]][:not-in [:lower :table_name]- [:raw- (str "("- (first (sql/format {:select [:index_name]- :from [[(t2/table-name :model/SearchIndexMetadata) :metadata]]- :where [:= :metadata.engine [:inline "appdb"]]}))- ")")]]]})))+ {:select [:%lower.index_name]+ :from [(t2/table-name :model/SearchIndexMetadata)]+ :where [:= :engine [:inline "appdb"]]}]]})))(defn- delete-obsolete-tables! [];; Delete metadata around indexes that are no longer needed.@@ -146,7 +141,8 @@(t2/query (sql.helpers/drop-table table))(vswap! dropped conj table);; Deletion could fail if it races with other instances- (catch ExceptionInfo _)))+ (catch Exception e+ (log/warnf e "Failed to drop stale index %s" table))))(log/infof "Dropped %d stale indexes: %s" (count @dropped) @dropped)))(defn- ->db-type [t]diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 4bad3a82392..6164d6afb75 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -205,12 +205,34 @@(-> (disj "indexed-entity") (concat ["indexed-entity"])))](reduce u/rconcat [] (map spec-index-reducible sorted-models))))+(def ^:private max-document-error-logs 10)+(defn- query->documents [query-reducible]- (->> query-reducible- (eduction- (comp- (map t2.realize/realize)- (map ->document)))))+ (let [failures (volatile! 0)]+ (->> query-reducible+ (eduction+ (comp+ (map t2.realize/realize)+ (fn [rf]+ (fn+ ([] (rf))+ ([result]+ (when (pos? @failures)+ (log/warnf "Failed to build %d search documents; they were skipped" @failures))+ (rf result))+ ([result m]+ (if-let [doc (try+ (->document m)+ (catch Throwable t+ (let [n (vswap! failures inc)]+ (cond+ (<= n max-document-error-logs)+ (log/warnf t "Error building search document for %s %s; skipping" (:model m) (:id m))+ (= n (inc max-document-error-logs))+ (log/warnf "Suppressing further per-document error logs after %d failures" max-document-error-logs)))+ nil))]+ (rf result doc)+ result)))))))))(defn searchable-documents"Return all existing searchable documents from the database."