Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Searchable document content (#75841)

Nick Fitzpatrick0c145ac68bf5
diff --git a/dev/src/dev/module_viz.clj b/dev/src/dev/module_viz.clj
index 4bb95573a4a..fa8ab0f60d3 100644
--- a/dev/src/dev/module_viz.clj
+++ b/dev/src/dev/module_viz.clj
@@ -94,7 +94,6 @@
(defn reset-cache! [] (reset! cache nil))
-
(defn- build-cache
"Build the full modules cache — a map with keys :kondo, :module-nodes, :api-children, :all-edges,
:edges-by-mod. Called once on first request and memoized via the `cache` atom."
diff --git a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
index 4983efefdc8..6c66d81fd71 100644
--- a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
+++ b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
@@ -40,6 +40,7 @@ export const SearchInputContainer = styled.div<{
}
return css`
background-color: var(--mb-color-background_page-primary);
+
&:hover {
background-color: var(--mb-color-background_page-secondary);
}
diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.clj
index f0083358b9f..ce19a582abd 100644
--- a/src/metabase/documents/models/document.clj
+++ b/src/metabase/documents/models/document.clj
@@ -8,9 +8,11 @@
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.public-sharing.core :as public-sharing]
+ [metabase.search.config :as search.config]
[metabase.search.spec :as search.spec]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
+ [metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[methodical.core :as methodical]
@@ -114,6 +116,27 @@
;;; ----------------------------------------------- Search ----------------------------------------------------------
+(defn- document->search-text
+ "Extract the plain searchable text from a document's prose-mirror body for the search index.
+
+ Receives the raw `:document` value as it comes off the ingestion query (a JSON string).
+ Returns nil if it can't be parsed, so a malformed/oversized body never blocks the rest of the
+ document (e.g. its name) from being indexed."
+ [document]
+ (when document
+ (try
+ (-> (cond-> document (string? document) json/decode+kw)
+ prose-mirror/ast->text
+ not-empty)
+ (catch Throwable _ nil))))
+
+;; The legacy (in-place) search engine LIKE-matches the raw `:document` JSON in SQL, but scores results
+;; against this cleaned-up text. Extracting prose here means a query that only hits JSON structure
+;; (e.g. "paragraph") matches no real content and is correctly dropped as a non-match.
+(defmethod search.config/column->string [:document :document]
+ [value _model _column]
+ (or (document->search-text value) ""))
+
(search.spec/define-spec "document"
{:model :model/Document
:attrs {:archived true
@@ -124,7 +147,11 @@
:updated-at :updated_at
:last-viewed-at :last_viewed_at
:pinned [:> [:coalesce :collection_position [:inline 0]] [:inline 0]]}
- :search-terms [:name]
+ :search-terms {:name true
+ :document document->search-text}
+ ;; Document bodies are full-text searchable (via `document->search-text` above) but are
+ ;; deliberately excluded from semantic-search embeddings.
+ :embedding-exclude #{:document}
:joins {:collection [:model/Collection [:= :collection.id :this.collection_id]]}
:render-terms {:document-name :name
:document-id :id
diff --git a/src/metabase/documents/prose_mirror.clj b/src/metabase/documents/prose_mirror.clj
index 4cbe34f880f..63f93f979d8 100644
--- a/src/metabase/documents/prose_mirror.clj
+++ b/src/metabase/documents/prose_mirror.clj
@@ -1,6 +1,7 @@
(ns metabase.documents.prose-mirror
"Manipulate the prose mirror ast for documents"
(:require
+ [clojure.string :as str]
[clojure.walk :as walk]))
(def card-embed-type
@@ -56,6 +57,23 @@
(->> (tree-seq :content :content document)
(keep collector)))
+(defn ast->text
+ "Extract the concatenated user-visible text from a prose-mirror document AST (the value of a
+ document's `:document` field).
+
+ Walks every node, in document order, and joins:
+ - the `:text` of `text` nodes, and
+ - the `:label` attr of reference nodes (smart links, mentions) — the text the editor actually
+ renders in place of the node.
+
+ Nodes that render no inline prose (card embeds, layout containers) contribute nothing.
+ Returns a (possibly empty) string."
+ [ast]
+ (->> (tree-seq :content :content ast)
+ (mapcat (juxt :text (comp :label :attrs)))
+ (remove str/blank?)
+ (str/join " ")))
+
(defn card-ids
"Get all card-ids"
[document]
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 1d9abb13332..812be93eb3d 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -198,6 +198,7 @@
(dissoc
:all-scores
:dataset_query
+ :document
:relevant-scores
:collection_effective_ancestors
:collection_id
diff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.clj
index 5f45d441eee..b73a71499ab 100644
--- a/src/metabase/search/in_place/filter.clj
+++ b/src/metabase/search/in_place/filter.clj
@@ -30,6 +30,12 @@
(def ^:private true-clause [:inline [:= 1 1]])
(def ^:private false-clause [:inline [:= 0 1]])
+(def ^:private max-document-search-length
+ "Cap the number of characters of a document's prose-mirror body that the legacy engine LIKE-scans.
+ A leading-wildcard LIKE can't use an index, so this bounds the worst case for pathologically large
+ documents. Mirrors the appdb engine's `metabase.search.ingestion/max-searchable-value-length`."
+ 500000)
+
;; ------------------------------------------------------------------------------------------------;;
;; Required Filters ;
;; ------------------------------------------------------------------------------------------------;;
@@ -86,6 +92,10 @@
(= column (search.config/column-with-model-alias model :dataset_query)))
[:like [:lower :query_action.dataset_query] wildcarded-token]
+ (and (= model "document")
+ (= column (search.config/column-with-model-alias "document" :document)))
+ [:like [:lower [:left column [:inline max-document-search-length]]] wildcarded-token]
+
:else
[:like [:lower column] wildcarded-token])))))
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 96cc07cb685..90b64c37133 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -105,6 +105,8 @@
:table_description :text
;; returned for Metric, Segment, and Action
:database_id :integer
+ ;; returned for Document
+ :document :text
;; returned for Database and Table
:initial_sync_status :text
:database_name :text
@@ -270,7 +272,8 @@
;; exclude them from exact-match ranking (and to keep in-place ordering stable).
(remove #{:collection_authority_level :moderated_status
:initial_sync_status :pk_ref :location
- :collection_location :data_authority :data_layer}))
+ :collection_location :data_authority
+ :data_layer :document}))
case-clauses (as-> columns-to-search <>
(map (fn [col] [:like [:lower col] match]) <>)
(interleave <> (repeat [:inline 0]))
@@ -350,6 +353,11 @@
[_ _]
[:name])
+(defmethod searchable-columns "document"
+ [_ _]
+ [:name
+ :document])
+
(def ^:private default-columns
"Columns returned for all models."
[:id :name :description :archived :created_at :updated_at])
@@ -405,7 +413,7 @@
(defmethod columns-for-model "document"
[_]
- [:id :name :archived :created_at :updated_at :collection_id :creator_id])
+ [:id :name :archived :created_at :updated_at :collection_id :creator_id :document])
(defmethod columns-for-model "transform"
[_]
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 16a53fe1b48..e5a112ab9ca 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -75,10 +75,16 @@
Note: Unlike searchable-text, transformation functions in search-terms
(e.g., explode-camel-case) are NOT applied. Transformations like camel-case
- explosion are specific to full-text search optimization."
+ explosion are specific to full-text search optimization.
+
+ Fields listed in the spec's `:embedding-exclude` set are kept out of the
+ embedding text entirely (but remain in `searchable-text`)."
[m]
- (let [search-terms (:search-terms (search.spec/spec (:model m)))
- field-keys (cond-> search-terms (map? search-terms) keys)
+ (let [spec (search.spec/spec (:model m))
+ search-terms (:search-terms spec)
+ excluded (:embedding-exclude spec #{})
+ field-keys (->> (cond-> search-terms (map? search-terms) keys)
+ (remove excluded))
header (str "[" (:model m) "]")
fields (keep (fn [k]
(let [v (get m k)]
diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.clj
index 56fd9974f43..80948ffc007 100644
--- a/src/metabase/search/spec.clj
+++ b/src/metabase/search/spec.clj
@@ -92,8 +92,10 @@
;; - `metabase.search.impl/add-dataset-collection-hierarchy` reads `:collection_location` to hydrate
;; `:collection_effective_ancestors`, and the collection-result hydration path reads `:location` from
;; the toucan instance (which the render-term keeps populated).
+ ;; `:document` is the document model's prose-mirror body: it's indexed as searchable text (via
+ ;; ast->text) but the raw JSON should never be echoed back in the search response or bloat the index row.
;; `:data_layer` also stays IN: Metabot surfaces it on table results so the LLM sees a table's data layer.
- #{:pinned :view_count :last_viewed_at :native_query :dataset_query})
+ #{:pinned :view_count :last_viewed_at :native_query :dataset_query :document})
(def attr-types
"The abstract types of each attribute."
@@ -202,6 +204,7 @@
[:search-terms [:or
[:sequential {:min 1} :keyword]
[:map-of :keyword [:or fn? true?]]]]
+ [:embedding-exclude {:optional true} [:set :keyword]]
[:render-terms [:map-of NonAttrKey AttrValue]]
[:where {:optional true} vector?]
[:bookmark {:optional true} vector?]
@@ -403,7 +406,12 @@
Spec keys:
- `:model` - Toucan model keyword (required)
- `:attrs` - Map of search index attributes (required)
- - `:search-terms` - Vector of searchable text fields (required)
+ - `:search-terms` - Searchable text fields: a vector of column keywords, or a map of
+ column keyword to either `true` (use the raw value) or a transform fn applied for
+ full-text search (required)
+ - `:embedding-exclude` - Set of `:search-terms` keys to omit from the semantic-search
+ embedding text (they remain in full-text search). Use for fields whose raw value is
+ not suitable for semantic search.
- `:render-terms` - Additional attributes needed for display (required)
- `:visibility` - `:all` (default) or `:app-user` (non-sandboxed, non-impersonated users only)
- `:where` - HoneySQL where clause to filter indexed records
diff --git a/test/metabase/documents/prose_mirror_test.clj b/test/metabase/documents/prose_mirror_test.clj
index d9c4d8d45b7..514c0f3dd10 100644
--- a/test/metabase/documents/prose_mirror_test.clj
+++ b/test/metabase/documents/prose_mirror_test.clj
@@ -317,3 +317,30 @@
(:text %))
result (prose-mirror/collect-ast doc collector)]
(is (= ["first" "second" "third"] (vec result))))))
+
+(deftest ^:parallel ast->text-test
+ (testing "joins text from all text nodes across nested blocks, in document order"
+ (let [ast {:type "doc"
+ :content [{:type "heading" :content [{:type "text" :text "Title"}]}
+ {:type "paragraph" :content [{:type "text" :text "Hello"}
+ {:type "text" :text "world"}]}]}]
+ (is (= "Title Hello world" (prose-mirror/ast->text ast)))))
+ (testing "includes the visible label of reference nodes (smart links, mentions), in document order"
+ (let [ast {:type "doc"
+ :content [{:type "paragraph"
+ :content [{:type "text" :text "see"}
+ {:type prose-mirror/smart-link-type
+ :attrs {:model "card" :entityId "abc" :label "Orders Question"}}
+ {:type "text" :text "and"}
+ {:type "mention" :attrs {:id 7 :label "Jane Doe"}}]}]}]
+ (is (= "see Orders Question and Jane Doe" (prose-mirror/ast->text ast)))))
+ (testing "ignores nodes that render no inline prose (card embeds, label-less references, layout containers)"
+ (let [ast {:type "doc"
+ :content [{:type "paragraph" :content [{:type "text" :text "before"}]}
+ {:type prose-mirror/card-embed-type :attrs {:id 42}}
+ {:type prose-mirror/smart-link-type :attrs {:model "card" :entityId "abc"}}
+ {:type "paragraph" :content [{:type "text" :text "after"}]}]}]
+ (is (= "before after" (prose-mirror/ast->text ast)))))
+ (testing "returns an empty string for an empty document"
+ (is (= "" (prose-mirror/ast->text {:type "doc" :content []})))
+ (is (= "" (prose-mirror/ast->text nil)))))
diff --git a/test/metabase/documents/search_test.clj b/test/metabase/documents/search_test.clj
index cde58380d4e..0b2a92f1335 100644
--- a/test/metabase/documents/search_test.clj
+++ b/test/metabase/documents/search_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.documents.test-util :as documents.test-util]
[metabase.events.core :as events]
[metabase.permissions.core :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
@@ -12,16 +13,6 @@
[metabase.util :as u]
[toucan2.core :as t2]))
-(defn- text->prose-mirror-ast
- "Convert plain text to a ProseMirror AST structure."
- [text]
- (if (empty? text)
- {:type "doc" :content []}
- {:type "doc"
- :content [{:type "paragraph"
- :content [{:type "text"
- :text text}]}]}))
-
(use-fixtures :once (fixtures/initialize :db :test-users))
(defn card-with-name-and-query
@@ -39,7 +30,7 @@
regular-card-name (str card-name "-regular")]
(search.tu/with-temp-index-table
(mt/with-temp [:model/Document document {:name "Test Document"
- :document (text->prose-mirror-ast "")}
+ :document (documents.test-util/text->prose-mirror-ast "")}
:model/Card document-card (-> (card-with-name-and-query card-name)
(assoc :document_id (u/the-id document)))
:model/Card regular-card (card-with-name-and-query regular-card-name)]
@@ -59,14 +50,14 @@
:model/Collection {private-coll-id :id} {}
:model/Document {doc-in-public-coll :id} {:name "Public Document"
:collection_id coll-id
- :document (text->prose-mirror-ast "content")}
+ :document (documents.test-util/text->prose-mirror-ast "content")}
:model/Document {doc-in-private-coll :id} {:name "Private Document"
:collection_id private-coll-id
- :document (text->prose-mirror-ast "content")}
+ :document (documents.test-util/text->prose-mirror-ast "content")}
:model/Document {doc-archived :id} {:name "Archived Document"
:collection_id coll-id
:archived true
- :document (text->prose-mirror-ast "content")}]
+ :document (documents.test-util/text->prose-mirror-ast "content")}]
;; Give user read access to first collection only
(perms/grant-collection-read-permissions! (perms-group/all-users) coll-id)
(testing "Regular user sees only documents in accessible collections"
@@ -98,7 +89,7 @@
(deftest document-view-tracking-integration-test
(testing "Document view tracking integrates with search"
(mt/with-temp [:model/Document {doc-id :id} {:name "Viewed Document"
- :document (text->prose-mirror-ast "content")
+ :document (documents.test-util/text->prose-mirror-ast "content")
:view_count 0}]
(testing "Document has initial state"
(let [doc (t2/select-one :model/Document :id doc-id)]
@@ -135,3 +126,53 @@
(is (= doc-id (:id doc-result)))
;; User recency affects scoring
(is (some #(= "user-recency" (:name %)) (:scores doc-result))))))))))))
+
+(deftest document-content-search-test
+ (testing "Documents are searchable by their body content, not just their name (UXW-4199)"
+ ;; Both engines search document bodies: the appdb engine indexes clean text (via ast->text),
+ ;; the legacy in-place engine LIKE-matches the raw prose-mirror JSON.
+ (mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary"
+ :document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]
+ (search.tu/with-new-search-and-legacy-search
+ (testing "found by a term that appears only in the body"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "projections" :models "document")]
+ (is (contains? (set (map :id (:data results))) doc-id))))
+ (testing "still found by its name"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Annual" :models "document")]
+ (is (contains? (set (map :id (:data results))) doc-id))))
+ (testing "not matched by prose-mirror JSON structure keywords"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "paragraph" :models "document")]
+ (is (not (contains? (set (map :id (:data results))) doc-id)))))))))
+
+(deftest document-smart-link-label-search-test
+ (testing "Documents are searchable by the visible label of an embedded smart link (UXW-4199)"
+ (mt/with-temp [:model/Document {doc-id :id}
+ {:name "Reference Doc"
+ :document {:type "doc"
+ :content [{:type "paragraph"
+ :content [{:type "text" :text "see"}
+ {:type "smartLink"
+ :attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]
+ (search.tu/with-new-search-and-legacy-search
+ (testing "found by a term that appears only in the smart-link label"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Retention" :models "document")]
+ (is (contains? (set (map :id (:data results))) doc-id))))))))
+
+(deftest document-content-search-on-update-test
+ (testing "Editing a document's body re-indexes its content for search (UXW-4199)"
+ ;; The appdb engine indexes the :document column (extracting body text via ast->text at index time),
+ ;; so editing the body lands in (t2/changes instance) and triggers a realtime reindex.
+ (search.tu/with-temp-index-table
+ (mt/with-temp [:model/Document {doc-id :id} {:name "Quarterly Report"
+ :document (documents.test-util/text->prose-mirror-ast "initial draft contents")}]
+ (testing "found by an original body term before the edit"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "initial" :models "document")]
+ (is (contains? (set (map :id (:data results))) doc-id))))
+ (t2/update! :model/Document doc-id
+ {:document (documents.test-util/text->prose-mirror-ast "revised projections and forecasts")})
+ (testing "found by a term that appears only in the revised body"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "forecasts" :models "document")]
+ (is (contains? (set (map :id (:data results))) doc-id))))
+ (testing "no longer found by the replaced body term"
+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "initial" :models "document")]
+ (is (not (contains? (set (map :id (:data results))) doc-id)))))))))
diff --git a/test/metabase/documents/test_util.clj b/test/metabase/documents/test_util.clj
index 7cf84e6f752..b463dcf2fda 100644
--- a/test/metabase/documents/test_util.clj
+++ b/test/metabase/documents/test_util.clj
@@ -2,9 +2,11 @@
"Shared test utilities for document tests.")
(defn text->prose-mirror-ast
- "Convert plain text to a ProseMirror AST structure."
+ "Convert plain text to a ProseMirror AST structure. Empty text yields an empty document."
[text]
- {:type "doc"
- :content [{:type "paragraph"
- :content [{:type "text"
- :text text}]}]})
+ (if (empty? text)
+ {:type "doc" :content []}
+ {:type "doc"
+ :content [{:type "paragraph"
+ :content [{:type "text"
+ :text text}]}]}))
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 6d7519ae8c0..3f2ed8b55d4 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -84,7 +84,20 @@
(with-redefs [search.spec/spec spec-fn]
(is (= "[table]\nname: CamelCaseTest"
(#'search.ingestion/embeddable-text record))
- "Transformation functions should not be applied to embeddable text for semantic search")))))
+ "Transformation functions should not be applied to embeddable text for semantic search"))))
+ (testing "embeddable-text omits fields listed in :embedding-exclude"
+ (let [spec-fn (constantly {:search-terms {:name true :document str}
+ :embedding-exclude #{:document}})
+ record {:model "document"
+ :name "Q3 Planning"
+ :document "the full document body"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "[document]\nname: Q3 Planning"
+ (#'search.ingestion/embeddable-text record))
+ "Excluded fields must not appear in the semantic-search embedding text")
+ (is (= "Q3 Planning the full document body"
+ (#'search.ingestion/searchable-text record))
+ "Excluded fields remain in full-text searchable-text")))))
(deftest execute-all-function-attrs-test
(testing "function-attr returning a map merges its keys into the document"