Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Avoid invalid chars from keeping appdb reindex to complete (#73759)

Nathan Voxland7ef13612e632
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 671368f5a89..d704951f324 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -301,6 +301,12 @@
:labels [:model]})
(prometheus/counter :metabase-search/index-error
{:description "Number of errors encountered when indexing for search"})
+ (prometheus/counter :metabase-search/appdb-index-batches-skipped
+ {:description "Number of search index batches skipped because the upsert errored."
+ :labels [:table-type]})
+ (prometheus/counter :metabase-search/index-documents-skipped
+ {:description "Number of individual search documents skipped because building the document errored."
+ :labels [:model]})
(prometheus/counter :metabase-search/index-update-ms
{:description "Total number of ms updating the index"})
(prometheus/histogram :metabase-search/index-update-duration-ms
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 8ee38a990af..6bf7e40cdec 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -248,18 +248,27 @@
;; Did *we* do a rotation?
(boolean pending)))))
+(defn- strip-junk-chars
+ "Replace control characters (\\p{Cc}: C0 controls including \\t \\n \\r, DEL, C1 controls) and surrogate
+ code points (\\p{Cs}) with a single space so they act as token boundaries for full-text indexing instead
+ of accidentally fusing adjacent words. Postgres also outright rejects literal NUL (0x00) in text columns,
+ so this is required to keep reindex batches from aborting. Non-string values pass through unchanged."
+ [v]
+ (cond-> v (string? v) (str/replace #"(?U)[\p{Cc}\p{Cs}]" " ")))
+
(defn- document->entry [entity]
- (-> entity
- (select-keys (conj search.spec/attr-columns :model :display_data :legacy_input))
- (set/rename-keys {:id :model_id
- :created_at :model_created_at
- :updated_at :model_updated_at})
- (assoc :updated_at :%now)
- (update :display_data json/encode)
- ;; legacy_input is already JSON-encoded in ->document; encode only if it's still a map (e.g., in tests)
- (update :legacy_input #(if (string? %) % (json/encode %)))
- (dissoc :native_query)
- (merge (specialization/extra-entry-fields entity))))
+ (let [entity (update-vals entity strip-junk-chars)]
+ (-> entity
+ (select-keys (conj search.spec/attr-columns :model :display_data :legacy_input))
+ (set/rename-keys {:id :model_id
+ :created_at :model_created_at
+ :updated_at :model_updated_at})
+ (assoc :updated_at :%now)
+ (update :display_data json/encode)
+ ;; legacy_input is already JSON-encoded in ->document; encode only if it's still a map (e.g., in tests)
+ (update :legacy_input #(if (string? %) % (json/encode %)))
+ (dissoc :native_query)
+ (merge (specialization/extra-entry-fields entity)))))
(defn- table-not-found-exception? [e]
;; Use with care, obviously this can give false positives if used with a query that's *actually* malformed.
@@ -279,7 +288,10 @@
(defn- safe-batch-upsert!
"A version of batch-upsert! that no-ops for missing indexes, and handles stale index tracking metadata.
- Returns the name of the table that was written to, or nil if there is none being tracked.
+ Returns the name of the table that was written to, or nil if there is none being tracked, or nil
+ if the upsert failed for any other reason — in which case the failure is logged at ERROR and we
+ continue so the rest of the reindex can finish and activate whatever was successfully written.
+
We recover gracefully the first time if the tracking atom was stale, but do not check again on retry."
[table-type table-name-fn entries]
;; For convenience, no-op if we are not tracking any table.
@@ -287,17 +299,33 @@
(let [upsert! (fn [t] (specialization/batch-upsert! t entries) t)]
(try
(upsert! table-name)
+ (catch InterruptedException ie
+ (.interrupt (Thread/currentThread))
+ (throw ie))
(catch Exception e
- ;; Only suppress failures related to a legitimately non-existent table
- (if (or (not (table-not-found-exception? e)) (exists? table-name))
- (throw e)
+ ;; If the failure is a legitimately non-existent table, refresh tracking and retry once.
+ (if (and (table-not-found-exception? e) (not (exists? table-name)))
(when-let [refreshed-table-name (do (sync-tracking-atoms!) (table-name-fn))]
(if (= table-name refreshed-table-name)
(throw (ex-info "Currently tracked index does not exist" e {:table-name table-name}))
(try
(upsert! refreshed-table-name)
+ (catch InterruptedException ie
+ (.interrupt (Thread/currentThread))
+ (throw ie))
(catch Exception e2
- (retry-upsert-ex table-type table-name refreshed-table-name e e2)))))))))))
+ (if (table-not-found-exception? e2)
+ (throw (retry-upsert-ex table-type table-name refreshed-table-name e e2))
+ (do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})
+ (log/errorf (retry-upsert-ex table-type table-name refreshed-table-name e e2)
+ "Error upserting search index batch into %s table %s after refresh; skipping batch and continuing"
+ (name table-type) refreshed-table-name)
+ nil))))))
+ ;; Any other failure - log and continue so reindex can still finish.
+ (do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})
+ (log/errorf e "Error upserting search index batch into %s table %s; skipping batch and continuing"
+ (name table-type) table-name)
+ nil)))))))
(defn- batch-update!
"Create the given search index entries in bulk. Commits after each batch"
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 04475b43b27..dd3ab524e3b 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -61,6 +61,15 @@
(for [model (keys (search.spec/specifications))]
{:model model}))
+(defmethod analytics.core/known-labels :metabase-search/appdb-index-batches-skipped
+ [_]
+ [{:table-type :active} {:table-type :pending}])
+
+(defmethod analytics.core/known-labels :metabase-search/index-documents-skipped
+ [_]
+ (for [model (keys (search.spec/specifications))]
+ {:model model}))
+
(defmethod analytics.core/known-labels :metabase-search/engine-default
[_]
(analytics.core/known-labels :metabase-search/engine-active))
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 078731e2fc0..333604c9de8 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -221,7 +221,11 @@
([result m]
(if-let [doc (try
(->document m)
- (catch Throwable t
+ (catch InterruptedException ie
+ (.interrupt (Thread/currentThread))
+ (throw ie))
+ (catch Exception t
+ (analytics/inc! :metabase-search/index-documents-skipped {:model (:model m)})
(let [n (vswap! failures inc)]
(cond
(<= n max-document-error-logs)
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index a393c7c3ab7..9f66a81d15a 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -628,6 +628,66 @@
(t2/delete! :model/SearchIndexMetadata :version "pending-timeout-test")
(#'search.index/delete-obsolete-tables!))))))
+(deftest strip-junk-chars-test
+ (let [strip @#'search.index/strip-junk-chars]
+ (testing "non-string values pass through unchanged"
+ (is (= 42 (strip 42)))
+ (is (= :a-kw (strip :a-kw)))
+ (is (= nil (strip nil)))
+ (is (= [:|| "x"] (strip [:|| "x"]))))
+ (testing "NUL byte (0x00) is replaced with a space — Postgres rejects raw NUL in text columns"
+ (is (= "a b" (strip "ab"))))
+ (testing "C0 controls (BEL, BS, VT, FF, US) are replaced with spaces"
+ (is (= "a b c d e f"
+ (strip "abc d ef"))))
+ (testing "tab/newline/CR also become spaces (already whitespace for tsvector tokenization)"
+ (is (= "a b c d" (strip "a\tb\nc\rd"))))
+ (testing "DEL (0x7F) and C1 controls (0x80-0x9F) are replaced"
+ (is (= "a b c d" (strip "ab€cŸd"))))
+ (testing "unpaired surrogate code points are replaced"
+ (is (= "a b" (strip (str "a" (char 0xD800) "b")))))
+ (testing "ordinary text — letters, digits, punctuation, CJK, emoji — is untouched"
+ (is (= "Hello, world! 123 你好 🦄"
+ (strip "Hello, world! 123 你好 🦄"))))
+ (testing "BOM (U+FEFF) and zero-width joiner (U+200D — Cf class) are intentionally NOT stripped"
+ (is (= "ab‍c" (strip "ab‍c"))))
+ (testing "control chars at start/end of string"
+ (is (= " a " (strip "a"))))
+ (testing "consecutive control chars each become a space"
+ (is (= "a b" (strip "ab"))))))
+
+(deftest document->entry-junk-chars-test
+ ;; document->entry calls (specialization/extra-entry-fields entity), which dispatches on (mdb/db-type)
+ ;; and is only implemented for :postgres and :h2 — running this on MySQL/MariaDB hits the missing-impl error.
+ (when (#{:postgres :h2} (mdb/db-type))
+ (let [->entry @#'search.index/document->entry
+ FF (str (char 0x000C))
+ NUL (str (char 0x0000))
+ BEL (str (char 0x0007))
+ DEL (str (char 0x007F))
+ dirty-name (str "Title" FF "Sub" NUL "title" BEL)
+ dirty-st (str "line1" DEL "line2")
+ entity {:model "card"
+ :name dirty-name
+ :searchable_text dirty-st
+ :display_data {:name dirty-name}
+ :legacy_input {}
+ :archived false}
+ entry (->entry entity)]
+ (testing "top-level :name column has control chars replaced with spaces"
+ (is (= "Title Sub title " (:name entry))))
+ (testing "search_vector / extra fields built from the stripped entity contain no raw control chars"
+ (let [printed (pr-str (vals (select-keys entry [:search_vector :with_native_query_vector
+ :search_terms :native_search_terms])))]
+ (is (false? (.contains printed NUL)) "no NUL byte")
+ (is (false? (.contains printed BEL)) "no BEL")
+ (is (false? (.contains printed FF)) "no form feed")
+ (is (false? (.contains printed DEL)) "no DEL")))
+ (testing "display_data preserves original characters (value was a map at strip time, encoded after)"
+ (let [^String json-str (:display_data entry)]
+ (is (false? (.contains json-str ^CharSequence NUL)) "encoded JSON has no raw NUL")
+ (is (false? (.contains json-str ^CharSequence FF)) "encoded JSON has no raw form feed"))))))
+
(deftest when-index-created
(when (search/supports-index?)
(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]