The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Searchable document content (#75841)
Nick Fitzpatrick0c145ac68bf5
diff --git a/dev/src/dev/module_viz.clj b/dev/src/dev/module_viz.cljindex 4bb95573a4a..fa8ab0f60d3 100644--- a/dev/src/dev/module_viz.clj+++ b/dev/src/dev/module_viz.clj@@ -94,7 +94,6 @@(defn reset-cache! [] (reset! cache nil))-(defn- build-cache"Build the full modules cache — a map with keys :kondo, :module-nodes, :api-children, :all-edges,:edges-by-mod. Called once on first request and memoized via the `cache` atom."diff --git a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsxindex 4983efefdc8..6c66d81fd71 100644--- a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx+++ b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx@@ -40,6 +40,7 @@ export const SearchInputContainer = styled.div<{}return css`background-color: var(--mb-color-background_page-primary);+&:hover {background-color: var(--mb-color-background_page-secondary);}diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.cljindex f0083358b9f..ce19a582abd 100644--- a/src/metabase/documents/models/document.clj+++ b/src/metabase/documents/models/document.clj@@ -8,9 +8,11 @@[metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.public-sharing.core :as public-sharing]+ [metabase.search.config :as search.config][metabase.search.spec :as search.spec][metabase.util :as u][metabase.util.i18n :refer [deferred-tru]]+ [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][methodical.core :as methodical]@@ -114,6 +116,27 @@;;; ----------------------------------------------- Search ----------------------------------------------------------+(defn- document->search-text+ "Extract the plain searchable text from a document's prose-mirror body for the search index.++ Receives the raw `:document` value as it comes off the ingestion query (a JSON string).+ Returns nil if it can't be parsed, so a malformed/oversized body never blocks the rest of the+ document (e.g. its name) from being indexed."+ [document]+ (when document+ (try+ (-> (cond-> document (string? document) json/decode+kw)+ prose-mirror/ast->text+ not-empty)+ (catch Throwable _ nil))))++;; The legacy (in-place) search engine LIKE-matches the raw `:document` JSON in SQL, but scores results+;; against this cleaned-up text. Extracting prose here means a query that only hits JSON structure+;; (e.g. "paragraph") matches no real content and is correctly dropped as a non-match.+(defmethod search.config/column->string [:document :document]+ [value _model _column]+ (or (document->search-text value) ""))+(search.spec/define-spec "document"{:model :model/Document:attrs {:archived true@@ -124,7 +147,11 @@:updated-at :updated_at:last-viewed-at :last_viewed_at:pinned [:> [:coalesce :collection_position [:inline 0]] [:inline 0]]}- :search-terms [:name]+ :search-terms {:name true+ :document document->search-text}+ ;; Document bodies are full-text searchable (via `document->search-text` above) but are+ ;; deliberately excluded from semantic-search embeddings.+ :embedding-exclude #{:document}:joins {:collection [:model/Collection [:= :collection.id :this.collection_id]]}:render-terms {:document-name :name:document-id :iddiff --git a/src/metabase/documents/prose_mirror.clj b/src/metabase/documents/prose_mirror.cljindex 4cbe34f880f..63f93f979d8 100644--- a/src/metabase/documents/prose_mirror.clj+++ b/src/metabase/documents/prose_mirror.clj@@ -1,6 +1,7 @@(ns metabase.documents.prose-mirror"Manipulate the prose mirror ast for documents"(:require+ [clojure.string :as str][clojure.walk :as walk]))(def card-embed-type@@ -56,6 +57,23 @@(->> (tree-seq :content :content document)(keep collector)))+(defn ast->text+ "Extract the concatenated user-visible text from a prose-mirror document AST (the value of a+ document's `:document` field).++ Walks every node, in document order, and joins:+ - the `:text` of `text` nodes, and+ - the `:label` attr of reference nodes (smart links, mentions) — the text the editor actually+ renders in place of the node.++ Nodes that render no inline prose (card embeds, layout containers) contribute nothing.+ Returns a (possibly empty) string."+ [ast]+ (->> (tree-seq :content :content ast)+ (mapcat (juxt :text (comp :label :attrs)))+ (remove str/blank?)+ (str/join " ")))+(defn card-ids"Get all card-ids"[document]diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 1d9abb13332..812be93eb3d 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -198,6 +198,7 @@(dissoc:all-scores:dataset_query+ :document:relevant-scores:collection_effective_ancestors:collection_iddiff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.cljindex 5f45d441eee..b73a71499ab 100644--- a/src/metabase/search/in_place/filter.clj+++ b/src/metabase/search/in_place/filter.clj@@ -30,6 +30,12 @@(def ^:private true-clause [:inline [:= 1 1]])(def ^:private false-clause [:inline [:= 0 1]])+(def ^:private max-document-search-length+ "Cap the number of characters of a document's prose-mirror body that the legacy engine LIKE-scans.+ A leading-wildcard LIKE can't use an index, so this bounds the worst case for pathologically large+ documents. Mirrors the appdb engine's `metabase.search.ingestion/max-searchable-value-length`."+ 500000)+;; ------------------------------------------------------------------------------------------------;;;; Required Filters ;;; ------------------------------------------------------------------------------------------------;;@@ -86,6 +92,10 @@(= column (search.config/column-with-model-alias model :dataset_query)))[:like [:lower :query_action.dataset_query] wildcarded-token]+ (and (= model "document")+ (= column (search.config/column-with-model-alias "document" :document)))+ [:like [:lower [:left column [:inline max-document-search-length]]] wildcarded-token]+:else[:like [:lower column] wildcarded-token])))))diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex 96cc07cb685..90b64c37133 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -105,6 +105,8 @@:table_description :text;; returned for Metric, Segment, and Action:database_id :integer+ ;; returned for Document+ :document :text;; returned for Database and Table:initial_sync_status :text:database_name :text@@ -270,7 +272,8 @@;; exclude them from exact-match ranking (and to keep in-place ordering stable).(remove #{:collection_authority_level :moderated_status:initial_sync_status :pk_ref :location- :collection_location :data_authority :data_layer}))+ :collection_location :data_authority+ :data_layer :document}))case-clauses (as-> columns-to-search <>(map (fn [col] [:like [:lower col] match]) <>)(interleave <> (repeat [:inline 0]))@@ -350,6 +353,11 @@[_ _][:name])+(defmethod searchable-columns "document"+ [_ _]+ [:name+ :document])+(def ^:private default-columns"Columns returned for all models."[:id :name :description :archived :created_at :updated_at])@@ -405,7 +413,7 @@(defmethod columns-for-model "document"[_]- [:id :name :archived :created_at :updated_at :collection_id :creator_id])+ [:id :name :archived :created_at :updated_at :collection_id :creator_id :document])(defmethod columns-for-model "transform"[_]diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 16a53fe1b48..e5a112ab9ca 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -75,10 +75,16 @@Note: Unlike searchable-text, transformation functions in search-terms(e.g., explode-camel-case) are NOT applied. Transformations like camel-case- explosion are specific to full-text search optimization."+ explosion are specific to full-text search optimization.++ Fields listed in the spec's `:embedding-exclude` set are kept out of the+ embedding text entirely (but remain in `searchable-text`)."[m]- (let [search-terms (:search-terms (search.spec/spec (:model m)))- field-keys (cond-> search-terms (map? search-terms) keys)+ (let [spec (search.spec/spec (:model m))+ search-terms (:search-terms spec)+ excluded (:embedding-exclude spec #{})+ field-keys (->> (cond-> search-terms (map? search-terms) keys)+ (remove excluded))header (str "[" (:model m) "]")fields (keep (fn [k](let [v (get m k)]diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex 56fd9974f43..80948ffc007 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -92,8 +92,10 @@;; - `metabase.search.impl/add-dataset-collection-hierarchy` reads `:collection_location` to hydrate;; `:collection_effective_ancestors`, and the collection-result hydration path reads `:location` from;; the toucan instance (which the render-term keeps populated).+ ;; `:document` is the document model's prose-mirror body: it's indexed as searchable text (via+ ;; ast->text) but the raw JSON should never be echoed back in the search response or bloat the index row.;; `:data_layer` also stays IN: Metabot surfaces it on table results so the LLM sees a table's data layer.- #{:pinned :view_count :last_viewed_at :native_query :dataset_query})+ #{:pinned :view_count :last_viewed_at :native_query :dataset_query :document})(def attr-types"The abstract types of each attribute."@@ -202,6 +204,7 @@[:search-terms [:or[:sequential {:min 1} :keyword][:map-of :keyword [:or fn? true?]]]]+ [:embedding-exclude {:optional true} [:set :keyword]][:render-terms [:map-of NonAttrKey AttrValue]][:where {:optional true} vector?][:bookmark {:optional true} vector?]@@ -403,7 +406,12 @@Spec keys:- `:model` - Toucan model keyword (required)- `:attrs` - Map of search index attributes (required)- - `:search-terms` - Vector of searchable text fields (required)+ - `:search-terms` - Searchable text fields: a vector of column keywords, or a map of+ column keyword to either `true` (use the raw value) or a transform fn applied for+ full-text search (required)+ - `:embedding-exclude` - Set of `:search-terms` keys to omit from the semantic-search+ embedding text (they remain in full-text search). Use for fields whose raw value is+ not suitable for semantic search.- `:render-terms` - Additional attributes needed for display (required)- `:visibility` - `:all` (default) or `:app-user` (non-sandboxed, non-impersonated users only)- `:where` - HoneySQL where clause to filter indexed recordsdiff --git a/test/metabase/documents/prose_mirror_test.clj b/test/metabase/documents/prose_mirror_test.cljindex d9c4d8d45b7..514c0f3dd10 100644--- a/test/metabase/documents/prose_mirror_test.clj+++ b/test/metabase/documents/prose_mirror_test.clj@@ -317,3 +317,30 @@(:text %))result (prose-mirror/collect-ast doc collector)](is (= ["first" "second" "third"] (vec result))))))++(deftest ^:parallel ast->text-test+ (testing "joins text from all text nodes across nested blocks, in document order"+ (let [ast {:type "doc"+ :content [{:type "heading" :content [{:type "text" :text "Title"}]}+ {:type "paragraph" :content [{:type "text" :text "Hello"}+ {:type "text" :text "world"}]}]}]+ (is (= "Title Hello world" (prose-mirror/ast->text ast)))))+ (testing "includes the visible label of reference nodes (smart links, mentions), in document order"+ (let [ast {:type "doc"+ :content [{:type "paragraph"+ :content [{:type "text" :text "see"}+ {:type prose-mirror/smart-link-type+ :attrs {:model "card" :entityId "abc" :label "Orders Question"}}+ {:type "text" :text "and"}+ {:type "mention" :attrs {:id 7 :label "Jane Doe"}}]}]}]+ (is (= "see Orders Question and Jane Doe" (prose-mirror/ast->text ast)))))+ (testing "ignores nodes that render no inline prose (card embeds, label-less references, layout containers)"+ (let [ast {:type "doc"+ :content [{:type "paragraph" :content [{:type "text" :text "before"}]}+ {:type prose-mirror/card-embed-type :attrs {:id 42}}+ {:type prose-mirror/smart-link-type :attrs {:model "card" :entityId "abc"}}+ {:type "paragraph" :content [{:type "text" :text "after"}]}]}]+ (is (= "before after" (prose-mirror/ast->text ast)))))+ (testing "returns an empty string for an empty document"+ (is (= "" (prose-mirror/ast->text {:type "doc" :content []})))+ (is (= "" (prose-mirror/ast->text nil)))))diff --git a/test/metabase/documents/search_test.clj b/test/metabase/documents/search_test.cljindex cde58380d4e..0b2a92f1335 100644--- a/test/metabase/documents/search_test.clj+++ b/test/metabase/documents/search_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.documents.test-util :as documents.test-util][metabase.events.core :as events][metabase.permissions.core :as perms][metabase.permissions.models.permissions-group :as perms-group]@@ -12,16 +13,6 @@[metabase.util :as u][toucan2.core :as t2]))-(defn- text->prose-mirror-ast- "Convert plain text to a ProseMirror AST structure."- [text]- (if (empty? text)- {:type "doc" :content []}- {:type "doc"- :content [{:type "paragraph"- :content [{:type "text"- :text text}]}]}))-(use-fixtures :once (fixtures/initialize :db :test-users))(defn card-with-name-and-query@@ -39,7 +30,7 @@regular-card-name (str card-name "-regular")](search.tu/with-temp-index-table(mt/with-temp [:model/Document document {:name "Test Document"- :document (text->prose-mirror-ast "")}+ :document (documents.test-util/text->prose-mirror-ast "")}:model/Card document-card (-> (card-with-name-and-query card-name)(assoc :document_id (u/the-id document))):model/Card regular-card (card-with-name-and-query regular-card-name)]@@ -59,14 +50,14 @@:model/Collection {private-coll-id :id} {}:model/Document {doc-in-public-coll :id} {:name "Public Document":collection_id coll-id- :document (text->prose-mirror-ast "content")}+ :document (documents.test-util/text->prose-mirror-ast "content")}:model/Document {doc-in-private-coll :id} {:name "Private Document":collection_id private-coll-id- :document (text->prose-mirror-ast "content")}+ :document (documents.test-util/text->prose-mirror-ast "content")}:model/Document {doc-archived :id} {:name "Archived Document":collection_id coll-id:archived true- :document (text->prose-mirror-ast "content")}]+ :document (documents.test-util/text->prose-mirror-ast "content")}];; Give user read access to first collection only(perms/grant-collection-read-permissions! (perms-group/all-users) coll-id)(testing "Regular user sees only documents in accessible collections"@@ -98,7 +89,7 @@(deftest document-view-tracking-integration-test(testing "Document view tracking integrates with search"(mt/with-temp [:model/Document {doc-id :id} {:name "Viewed Document"- :document (text->prose-mirror-ast "content")+ :document (documents.test-util/text->prose-mirror-ast "content"):view_count 0}](testing "Document has initial state"(let [doc (t2/select-one :model/Document :id doc-id)]@@ -135,3 +126,53 @@(is (= doc-id (:id doc-result)));; User recency affects scoring(is (some #(= "user-recency" (:name %)) (:scores doc-result))))))))))))++(deftest document-content-search-test+ (testing "Documents are searchable by their body content, not just their name (UXW-4199)"+ ;; Both engines search document bodies: the appdb engine indexes clean text (via ast->text),+ ;; the legacy in-place engine LIKE-matches the raw prose-mirror JSON.+ (mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary"+ :document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]+ (search.tu/with-new-search-and-legacy-search+ (testing "found by a term that appears only in the body"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "projections" :models "document")]+ (is (contains? (set (map :id (:data results))) doc-id))))+ (testing "still found by its name"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Annual" :models "document")]+ (is (contains? (set (map :id (:data results))) doc-id))))+ (testing "not matched by prose-mirror JSON structure keywords"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "paragraph" :models "document")]+ (is (not (contains? (set (map :id (:data results))) doc-id)))))))))++(deftest document-smart-link-label-search-test+ (testing "Documents are searchable by the visible label of an embedded smart link (UXW-4199)"+ (mt/with-temp [:model/Document {doc-id :id}+ {:name "Reference Doc"+ :document {:type "doc"+ :content [{:type "paragraph"+ :content [{:type "text" :text "see"}+ {:type "smartLink"+ :attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]+ (search.tu/with-new-search-and-legacy-search+ (testing "found by a term that appears only in the smart-link label"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Retention" :models "document")]+ (is (contains? (set (map :id (:data results))) doc-id))))))))++(deftest document-content-search-on-update-test+ (testing "Editing a document's body re-indexes its content for search (UXW-4199)"+ ;; The appdb engine indexes the :document column (extracting body text via ast->text at index time),+ ;; so editing the body lands in (t2/changes instance) and triggers a realtime reindex.+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Document {doc-id :id} {:name "Quarterly Report"+ :document (documents.test-util/text->prose-mirror-ast "initial draft contents")}]+ (testing "found by an original body term before the edit"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "initial" :models "document")]+ (is (contains? (set (map :id (:data results))) doc-id))))+ (t2/update! :model/Document doc-id+ {:document (documents.test-util/text->prose-mirror-ast "revised projections and forecasts")})+ (testing "found by a term that appears only in the revised body"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "forecasts" :models "document")]+ (is (contains? (set (map :id (:data results))) doc-id))))+ (testing "no longer found by the replaced body term"+ (let [results (mt/user-http-request :crowberto :get 200 "search" :q "initial" :models "document")]+ (is (not (contains? (set (map :id (:data results))) doc-id)))))))))diff --git a/test/metabase/documents/test_util.clj b/test/metabase/documents/test_util.cljindex 7cf84e6f752..b463dcf2fda 100644--- a/test/metabase/documents/test_util.clj+++ b/test/metabase/documents/test_util.clj@@ -2,9 +2,11 @@"Shared test utilities for document tests.")(defn text->prose-mirror-ast- "Convert plain text to a ProseMirror AST structure."+ "Convert plain text to a ProseMirror AST structure. Empty text yields an empty document."[text]- {:type "doc"- :content [{:type "paragraph"- :content [{:type "text"- :text text}]}]})+ (if (empty? text)+ {:type "doc" :content []}+ {:type "doc"+ :content [{:type "paragraph"+ :content [{:type "text"+ :text text}]}]}))diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 6d7519ae8c0..3f2ed8b55d4 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -84,7 +84,20 @@(with-redefs [search.spec/spec spec-fn](is (= "[table]\nname: CamelCaseTest"(#'search.ingestion/embeddable-text record))- "Transformation functions should not be applied to embeddable text for semantic search")))))+ "Transformation functions should not be applied to embeddable text for semantic search"))))+ (testing "embeddable-text omits fields listed in :embedding-exclude"+ (let [spec-fn (constantly {:search-terms {:name true :document str}+ :embedding-exclude #{:document}})+ record {:model "document"+ :name "Q3 Planning"+ :document "the full document body"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "[document]\nname: Q3 Planning"+ (#'search.ingestion/embeddable-text record))+ "Excluded fields must not appear in the semantic-search embedding text")+ (is (= "Q3 Planning the full document body"+ (#'search.ingestion/searchable-text record))+ "Excluded fields remain in full-text searchable-text")))))(deftest execute-all-function-attrs-test(testing "function-attr returning a map merges its keys into the document"