The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Avoid invalid chars from keeping appdb reindex to complete (#73759)
Nathan Voxland7ef13612e632
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 671368f5a89..d704951f324 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -301,6 +301,12 @@:labels [:model]})(prometheus/counter :metabase-search/index-error{:description "Number of errors encountered when indexing for search"})+ (prometheus/counter :metabase-search/appdb-index-batches-skipped+ {:description "Number of search index batches skipped because the upsert errored."+ :labels [:table-type]})+ (prometheus/counter :metabase-search/index-documents-skipped+ {:description "Number of individual search documents skipped because building the document errored."+ :labels [:model]})(prometheus/counter :metabase-search/index-update-ms{:description "Total number of ms updating the index"})(prometheus/histogram :metabase-search/index-update-duration-msdiff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex 8ee38a990af..6bf7e40cdec 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -248,18 +248,27 @@;; Did *we* do a rotation?(boolean pending)))))+(defn- strip-junk-chars+ "Replace control characters (\\p{Cc}: C0 controls including \\t \\n \\r, DEL, C1 controls) and surrogate+ code points (\\p{Cs}) with a single space so they act as token boundaries for full-text indexing instead+ of accidentally fusing adjacent words. Postgres also outright rejects literal NUL (0x00) in text columns,+ so this is required to keep reindex batches from aborting. Non-string values pass through unchanged."+ [v]+ (cond-> v (string? v) (str/replace #"(?U)[\p{Cc}\p{Cs}]" " ")))+(defn- document->entry [entity]- (-> entity- (select-keys (conj search.spec/attr-columns :model :display_data :legacy_input))- (set/rename-keys {:id :model_id- :created_at :model_created_at- :updated_at :model_updated_at})- (assoc :updated_at :%now)- (update :display_data json/encode)- ;; legacy_input is already JSON-encoded in ->document; encode only if it's still a map (e.g., in tests)- (update :legacy_input #(if (string? %) % (json/encode %)))- (dissoc :native_query)- (merge (specialization/extra-entry-fields entity))))+ (let [entity (update-vals entity strip-junk-chars)]+ (-> entity+ (select-keys (conj search.spec/attr-columns :model :display_data :legacy_input))+ (set/rename-keys {:id :model_id+ :created_at :model_created_at+ :updated_at :model_updated_at})+ (assoc :updated_at :%now)+ (update :display_data json/encode)+ ;; legacy_input is already JSON-encoded in ->document; encode only if it's still a map (e.g., in tests)+ (update :legacy_input #(if (string? %) % (json/encode %)))+ (dissoc :native_query)+ (merge (specialization/extra-entry-fields entity)))))(defn- table-not-found-exception? [e];; Use with care, obviously this can give false positives if used with a query that's *actually* malformed.@@ -279,7 +288,10 @@(defn- safe-batch-upsert!"A version of batch-upsert! that no-ops for missing indexes, and handles stale index tracking metadata.- Returns the name of the table that was written to, or nil if there is none being tracked.+ Returns the name of the table that was written to, or nil if there is none being tracked, or nil+ if the upsert failed for any other reason — in which case the failure is logged at ERROR and we+ continue so the rest of the reindex can finish and activate whatever was successfully written.+We recover gracefully the first time if the tracking atom was stale, but do not check again on retry."[table-type table-name-fn entries];; For convenience, no-op if we are not tracking any table.@@ -287,17 +299,33 @@(let [upsert! (fn [t] (specialization/batch-upsert! t entries) t)](try(upsert! table-name)+ (catch InterruptedException ie+ (.interrupt (Thread/currentThread))+ (throw ie))(catch Exception e- ;; Only suppress failures related to a legitimately non-existent table- (if (or (not (table-not-found-exception? e)) (exists? table-name))- (throw e)+ ;; If the failure is a legitimately non-existent table, refresh tracking and retry once.+ (if (and (table-not-found-exception? e) (not (exists? table-name)))(when-let [refreshed-table-name (do (sync-tracking-atoms!) (table-name-fn))](if (= table-name refreshed-table-name)(throw (ex-info "Currently tracked index does not exist" e {:table-name table-name}))(try(upsert! refreshed-table-name)+ (catch InterruptedException ie+ (.interrupt (Thread/currentThread))+ (throw ie))(catch Exception e2- (retry-upsert-ex table-type table-name refreshed-table-name e e2)))))))))))+ (if (table-not-found-exception? e2)+ (throw (retry-upsert-ex table-type table-name refreshed-table-name e e2))+ (do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})+ (log/errorf (retry-upsert-ex table-type table-name refreshed-table-name e e2)+ "Error upserting search index batch into %s table %s after refresh; skipping batch and continuing"+ (name table-type) refreshed-table-name)+ nil))))))+ ;; Any other failure - log and continue so reindex can still finish.+ (do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})+ (log/errorf e "Error upserting search index batch into %s table %s; skipping batch and continuing"+ (name table-type) table-name)+ nil)))))))(defn- batch-update!"Create the given search index entries in bulk. Commits after each batch"diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 04475b43b27..dd3ab524e3b 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -61,6 +61,15 @@(for [model (keys (search.spec/specifications))]{:model model}))+(defmethod analytics.core/known-labels :metabase-search/appdb-index-batches-skipped+ [_]+ [{:table-type :active} {:table-type :pending}])++(defmethod analytics.core/known-labels :metabase-search/index-documents-skipped+ [_]+ (for [model (keys (search.spec/specifications))]+ {:model model}))+(defmethod analytics.core/known-labels :metabase-search/engine-default[_](analytics.core/known-labels :metabase-search/engine-active))diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 078731e2fc0..333604c9de8 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -221,7 +221,11 @@([result m](if-let [doc (try(->document m)- (catch Throwable t+ (catch InterruptedException ie+ (.interrupt (Thread/currentThread))+ (throw ie))+ (catch Exception t+ (analytics/inc! :metabase-search/index-documents-skipped {:model (:model m)})(let [n (vswap! failures inc)](cond(<= n max-document-error-logs)diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex a393c7c3ab7..9f66a81d15a 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -628,6 +628,66 @@(t2/delete! :model/SearchIndexMetadata :version "pending-timeout-test")(#'search.index/delete-obsolete-tables!))))))+(deftest strip-junk-chars-test+ (let [strip @#'search.index/strip-junk-chars]+ (testing "non-string values pass through unchanged"+ (is (= 42 (strip 42)))+ (is (= :a-kw (strip :a-kw)))+ (is (= nil (strip nil)))+ (is (= [:|| "x"] (strip [:|| "x"]))))+ (testing "NUL byte (0x00) is replaced with a space — Postgres rejects raw NUL in text columns"+ (is (= "a b" (strip "a b"))))+ (testing "C0 controls (BEL, BS, VT, FF, US) are replaced with spaces"+ (is (= "a b c d e f"+ (strip "abcdef"))))+ (testing "tab/newline/CR also become spaces (already whitespace for tsvector tokenization)"+ (is (= "a b c d" (strip "a\tb\nc\rd"))))+ (testing "DEL (0x7F) and C1 controls (0x80-0x9F) are replaced"+ (is (= "a b c d" (strip "abcd"))))+ (testing "unpaired surrogate code points are replaced"+ (is (= "a b" (strip (str "a" (char 0xD800) "b")))))+ (testing "ordinary text — letters, digits, punctuation, CJK, emoji — is untouched"+ (is (= "Hello, world! 123 你好 🦄"+ (strip "Hello, world! 123 你好 🦄"))))+ (testing "BOM (U+FEFF) and zero-width joiner (U+200D — Cf class) are intentionally NOT stripped"+ (is (= "abc" (strip "abc"))))+ (testing "control chars at start/end of string"+ (is (= " a " (strip " a"))))+ (testing "consecutive control chars each become a space"+ (is (= "a b" (strip "a b"))))))++(deftest document->entry-junk-chars-test+ ;; document->entry calls (specialization/extra-entry-fields entity), which dispatches on (mdb/db-type)+ ;; and is only implemented for :postgres and :h2 — running this on MySQL/MariaDB hits the missing-impl error.+ (when (#{:postgres :h2} (mdb/db-type))+ (let [->entry @#'search.index/document->entry+ FF (str (char 0x000C))+ NUL (str (char 0x0000))+ BEL (str (char 0x0007))+ DEL (str (char 0x007F))+ dirty-name (str "Title" FF "Sub" NUL "title" BEL)+ dirty-st (str "line1" DEL "line2")+ entity {:model "card"+ :name dirty-name+ :searchable_text dirty-st+ :display_data {:name dirty-name}+ :legacy_input {}+ :archived false}+ entry (->entry entity)]+ (testing "top-level :name column has control chars replaced with spaces"+ (is (= "Title Sub title " (:name entry))))+ (testing "search_vector / extra fields built from the stripped entity contain no raw control chars"+ (let [printed (pr-str (vals (select-keys entry [:search_vector :with_native_query_vector+ :search_terms :native_search_terms])))]+ (is (false? (.contains printed NUL)) "no NUL byte")+ (is (false? (.contains printed BEL)) "no BEL")+ (is (false? (.contains printed FF)) "no form feed")+ (is (false? (.contains printed DEL)) "no DEL")))+ (testing "display_data preserves original characters (value was a map at strip time, encoded after)"+ (let [^String json-str (:display_data entry)]+ (is (false? (.contains json-str ^CharSequence NUL)) "encoded JSON has no raw NUL")+ (is (false? (.contains json-str ^CharSequence FF)) "encoded JSON has no raw form feed"))))))+(deftest when-index-created(when (search/supports-index?)(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]