The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[semantic search] Separate out embeddable_text from searchable_text and use labeled format for embedding (#64693)
Noah Mossb4c65c32993e
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex fba232ac253..dfec2274ead 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -118,7 +118,7 @@(defn- doc->db-record"Convert a document to a database record with a provided embedding."- [embedding-vec {:keys [model id searchable_text created_at creator_id updated_at+ [embedding-vec {:keys [model id searchable_text embeddable_text native_query created_at creator_id updated_atlast_editor_id archived verified official_collection database_id collection_id display_type legacy_inputpinned dashboardcard_count view_count last_viewed_at] :as doc}]{:model model@@ -128,7 +128,7 @@:database_id database_id:last_editor_id last_editor_id:name (or (:name doc) "")- :content searchable_text+ :content embeddable_text:display_type display_type:archived (some-> archived to-boolean):official_collection (some-> official_collection to-boolean)@@ -145,18 +145,18 @@:text_search_vector (if (:name doc)[:||(search/weighted-tsvector "A" (:name doc))- (search/weighted-tsvector "B" (or (:searchable_text doc) ""))]- (search/weighted-tsvector "A" (or (:searchable_text doc) "")))+ (search/weighted-tsvector "B" (or searchable_text ""))]+ (search/weighted-tsvector "A" (or searchable_text ""))):text_search_with_native_query_vector(if (:name doc)[:||(search/weighted-tsvector "A" (:name doc))(search/weighted-tsvector "B"- (str/join " " (remove str/blank? [(or (:searchable_text doc) "")- (or (:native_query doc) "")])))]+ (str/join " " (remove str/blank? [(or searchable_text "")+ (or native_query "")])))](search/weighted-tsvector "A"- (str/join " " (remove str/blank? [(or (:searchable_text doc) "")- (or (:native_query doc) "")]))))})+ (str/join " " (remove str/blank? [(or searchable_text "")+ (or native_query "")]))))})(defn index-size"Fetches the number of documents in the index table."@@ -299,14 +299,14 @@(defn- upsert-index-batch- (let [text->docs (group-by :searchable_text documents)- searchable-texts (keys text->docs)+ (let [text->docs (group-by :embeddable_text documents)+ embeddable-texts (keys text->docs)upsert-embedding! (upsert-embedding!-fn connectable index text->docs)[new-texts stats]- (u/profile (str "Semantic search embedding caching attempt for " {:docs (count documents) :texts (count searchable-texts)})- (let [[new-texts existing-embeddings] (partition-existing-embeddings connectable index searchable-texts)]+ (u/profile (str "Semantic search embedding caching attempt for " {:docs (count documents) :texts (count embeddable-texts)})+ (let [[new-texts existing-embeddings] (partition-existing-embeddings connectable index embeddable-texts)](if-not (seq existing-embeddings)- [searchable-texts nil]+ [embeddable-texts nil](u/profile (str "Semantic search cached embedding db update for " {:texts (count existing-embeddings)})[new-texts (upsert-embedding! existing-embeddings)]))))](->>diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.cljindex 7b7c5d9849f..2ebca765388 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj@@ -137,8 +137,8 @@t1 (ts "2025-01-01T10:00:00Z")t2 (ts "2025-01-01T11:00:00Z")t3 (ts "2025-01-01T12:00:00Z")- c1 {:model "card" :id "1" :name "Test" :searchable_text "Content"}- c2 {:model "card" :id "2" :name "Test" :searchable_text "Content"}+ c1 {:model "card" :id "1" :name "Test" :searchable_text "Content" :embeddable_text "Content"}+ c2 {:model "card" :id "2" :name "Test" :searchable_text "Content" :embeddable_text "Content"}version semantic.gate/search-doc->gate-docdelete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))]@@ -202,8 +202,8 @@clock-ref (volatile! (Instant/parse "2025-01-04T00:00:00Z"))clock (reify InstantSource (instant [_] @clock-ref))t1 (ts "2025-01-01T10:00:00Z")- c1 {:model "card" :id "1" :name "Test" :searchable_text "Content"}- c2 {:model "card" :id "2" :name "Test" :searchable_text "Content"}+ c1 {:model "card" :id "1" :name "Test" :searchable_text "Content" :embeddable_text "Content"}+ c2 {:model "card" :id "2" :name "Test" :searchable_text "Content" :embeddable_text "Content"}version semantic.gate/search-doc->gate-docadd-gate-to-dlq! (fn [pgvector index-metadata index-id](->> (semantic.gate/poll pgvector index-metadata {})@@ -309,7 +309,8 @@(.setValue (json/encode {:model "card":name "hey":id "1"- :searchable_text "foo"})))+ :searchable_text "foo"+ :embeddable_text "foo"}))):gated_at (ts "2025-01-04T09:00:00Z"):error_gated_at (ts "2025-01-04T09:00:00Z")}{:id "card_2"diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.cljindex 06b5aea0109..96ecb3a4694 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj@@ -37,6 +37,7 @@search-doc {:model "card":id "123":searchable_text "Dog Training Guide"+ :embeddable_text "Dog Training Guide":updated_at t1}sut semantic.gate/search-doc->gate-doc](is (= {:id "card_123"@@ -55,7 +56,7 @@(:updated_at (sut (assoc search-doc :updated_at nil) t2))))))))(deftest gate-doc->search-doc-test- (let [original-search-doc {:model "card" :id "123" :searchable_text "Dog Training Guide"}+ (let [original-search-doc {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}gate-doc {:document (doto (PGobject.)(.setType "jsonb")(.setValue (json/encode original-search-doc)))}@@ -96,10 +97,10 @@t1 (ts "2025-01-01T00:01:00Z")t2 (ts "2025-01-02T00:03:10Z")t3 (ts "2025-01-03T00:02:42Z")- c1 {:model "card" :id "123" :searchable_text "Dog Training Guide"}- c2 {:model "card" :id "123" :searchable_text "Dog Training Guide 2"}- c3 {:model "card" :id "123" :searchable_text "Dog Training Guide 3"}- d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration"}+ c1 {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}+ c2 {:model "card" :id "123" :searchable_text "Dog Training Guide 2" :embeddable_text "Dog Training Guide 2"}+ c3 {:model "card" :id "123" :searchable_text "Dog Training Guide 3" :embeddable_text "Dog Training Guide 3"}+ d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration" :embeddable_text "Elephant Migration"}version semantic.gate/search-doc->gate-docdelete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))sut semantic.gate/gate-documents!]@@ -172,9 +173,9 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)epoch-watermark {:last-poll Instant/EPOCH :last-seen Instant/EPOCH}- c1 {:model "card" :id "123" :searchable_text "a"}- c2 {:model "card" :id "234" :searchable_text "b"}- c3 {:model "card" :id "345" :searchable_text "c"}+ c1 {:model "card" :id "123" :searchable_text "a" :embeddable_text "a"}+ c2 {:model "card" :id "234" :searchable_text "b" :embeddable_text "b"}+ c3 {:model "card" :id "345" :searchable_text "c" :embeddable_text "c"}t0 (ts "2025-01-01T00:00:00Z")t1 (ts "2025-01-01T00:01:00Z")t2 (ts "2025-01-02T00:03:10Z")@@ -334,8 +335,8 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)t1 (ts "2025-01-01T00:01:00Z")- c1 {:model "card" :id "123" :searchable_text "Dog Training Guide"}- d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration"}+ c1 {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}+ d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration" :embeddable_text "Elephant Migration"}version semantic.gate/search-doc->gate-docsut semantic.gate/gate-documents!](with-open [_ (open-tables! pgvector index-metadata)]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 99bbdbde9d3..e8342ab8ba3 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -213,6 +213,7 @@:id "1":name "Dog Training Guide":searchable_text "Dog Training Guide"+ :embeddable_text "Dog Training Guide":creator_id 1:legacy_input {:model "card" :id "1"}:metadata {}}@@ -220,6 +221,7 @@:id "2":name "Elephant Migration":searchable_text "Elephant Migration"+ :embeddable_text "Elephant Migration":creator_id 2:legacy_input {:model "card" :id "2"}:metadata {}}@@ -227,6 +229,7 @@:id "3":name "Tiger Conservation":searchable_text "Tiger Conservation"+ :embeddable_text "Tiger Conservation":creator_id 3:legacy_input {:model "card" :id "3"}:metadata {}}]]@@ -255,6 +258,7 @@:id "1":name "Dog Training Guide":searchable_text "Dog Training Guide"+ :embeddable_text "Dog Training Guide":creator_id 1:legacy_input {:model "card" :id "1"}:metadata {}}@@ -262,6 +266,7 @@:id "2":name "Elephant Migration":searchable_text "Elephant Migration"+ :embeddable_text "Elephant Migration":creator_id 2:legacy_input {:model "card" :id "2"}:metadata {}}@@ -269,6 +274,7 @@:id "3":name "Dog Training Guide":searchable_text "Dog Training Guide"+ :embeddable_text "Dog Training Guide":creator_id 3:legacy_input {:model "card" :id "3"}:metadata {}}]]@@ -533,6 +539,7 @@base-doc {:model "card":id "123":searchable_text "test content"+ :embeddable_text "test content":creator_id 1}](testing "MySQL-style integer booleans are converted to real booleans"@@ -587,6 +594,7 @@:id (:id model-1):name (:name model-1):searchable_text (:name model-1)+ :embeddable_text (:name model-1):created_at #t "2025-01-01T12:00:00Z":creator_id (mt/user->id :crowberto):archived false@@ -601,6 +609,7 @@:archived false:collection_id coll-id:searchable_text "Antarctic wildlife"+ :embeddable_text "Antarctic wildlife":legacy_input {:id (str (:id model-index-1) ":1234"):name "Antarctic wildlife":model "indexed-entity"}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.cljindex 176e714812d..335bef386f8 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj@@ -41,9 +41,9 @@t1 (ts "2025-01-01T00:01:00Z")t2 (ts "2025-01-02T00:03:10Z")t3 (ts "2025-01-03T00:02:42Z")- c1 {:model "card" :id "1" :name "Poodle" :searchable_text "Dog Training Guide"}- c2 {:model "card" :id "2" :name "Pug" :searchable_text "Dog Training Guide 2"}- c3 {:model "card" :id "3" :name "Collie" :searchable_text "Dog Training Guide 3"}+ c1 {:model "card" :id "1" :name "Poodle" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}+ c2 {:model "card" :id "2" :name "Pug" :searchable_text "Dog Training Guide 2" :embeddable_text "Dog Training Guide 2"}+ c3 {:model "card" :id "3" :name "Collie" :searchable_text "Dog Training Guide 3" :embeddable_text "Dog Training Guide 3"}version semantic.gate/search-doc->gate-docdelete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))](with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)@@ -289,7 +289,7 @@model semantic.tu/mock-embedding-modelindex (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)t1 (ts "2025-01-01T00:01:00Z")- c1 {:model "card" :id "1" :name "Dog" :searchable_text "Dog Training Guide"}+ c1 {:model "card" :id "1" :name "Dog" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}version semantic.gate/search-doc->gate-doc](with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)_ (semantic.tu/open-index! pgvector index)]@@ -549,7 +549,7 @@clock-ref (volatile! (Instant/parse "2025-01-04T10:00:00Z"))clock (reify InstantSource (instant [_] @clock-ref))t1 (ts "2025-01-01T00:01:00Z")- card (fn [id] {:model "card" :id (str id) :name "Test" :searchable_text "Content"})+ card (fn [id] {:model "card" :id (str id) :name "Test" :searchable_text "Content" :embeddable_text "Content"})version semantic.gate/search-doc->gate-docfresh-indexing-state (fn [](let [state (semantic.indexer/init-indexing-state (get-metadata-row! pgvector index-metadata index))]@@ -668,7 +668,7 @@clock-ref (volatile! (Instant/parse "2025-01-04T10:00:00Z"))clock (reify InstantSource (instant [_] @clock-ref))t1 (ts "2025-01-01T00:01:00Z")- card (fn [id content] {:model "card" :id (str id) :name content :searchable_text content})+ card (fn [id content] {:model "card" :id (str id) :name content :searchable_text content :embeddable_text content})version semantic.gate/search-doc->gate-docpoisoned-doc-id (volatile! nil)get-indexed-docs (fn []diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.cljindex 5e89d435c50..dcec06566b7 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj@@ -173,7 +173,7 @@remove-scores (fn [rows] (mapv #(dissoc % :score :all-scores) rows)) ; scores have time-sensitives componentssut* semantic.pgvector-api/querysut #(remove-scores (:results (mt/as-admin (apply sut* %&)))) ; see notes below about perms- search-string "insect patterns" ; specifics of search will be handled under index tests+ search-string "puppy" ; specifics of search will be handled under index tests_ (assert (semantic.tu/mock-embeddings search-string)"search string should have test embedding")search {:search-string search-string}]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.cljindex 13285d4f37b..960a1160190 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj@@ -34,6 +34,7 @@:id (str id):name name:searchable_text name+ :embeddable_text name:updated_at (t/instant):creator_id 1})diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.cljindex b5436bc6543..5a882bb8ba7 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj@@ -19,6 +19,7 @@[doc](merge {:id 123:searchable_text (:name doc)+ :embeddable_text (:name doc):created_at #t "2025-01-01T12:00:00Z":creator_id (mt/user->id :rasta):archived false@@ -70,9 +71,9 @@(with-index-contents)+(def ^:private mock-documents-for-embeddings+ "Mock documents and queries for testing, associated with fake embedding vectors in [[mock-embeddings]]."+ [{:model "card" :name "Dog Training Guide" :query "puppy"}+ {:model "card" :name "Bird Watching Tips" :query "avian"}+ {:model "card" :name "Cat Behavior Study" :query "feline"}+ {:model "card" :name "Horse Racing Analysis" :query "equine"}+ {:model "card" :name "Fish Tank Setup" :query "aquatic"}+ {:model "card" :name "Bigfoot Sightings" :query "spooky video evidence"}+ {:model "dashboard" :name "Elephant Migration" :query "pachyderm"}+ {:model "dashboard" :name "Lion Pride Dynamics" :query "predator"}+ {:model "dashboard" :name "Penguin Colony Study" :query "Antarctic wildlife"}+ {:model "dashboard" :name "Whale Communication" :query "marine mammal"}+ {:model "dashboard" :name "Tiger Conservation" :query "endangered species"}+ {:model "dashboard" :name "Loch Ness Stuff" :query "prehistoric monsters"}+ ;; Tables include display_name in their search-terms (see table.clj:387-389), which gets+ ;; included in embeddable_text as "display_name: Species Table". We must include it here+ ;; so the mock embeddings match what gets indexed.+ {:model "table" :name "Species Table" :display_name "Species Table" :query "taxonomy"}+ {:model "table" :name "Monsters Table" :display_name "Monsters Table" :query "monster facts"}])++(def ^:private base-embedding-vectors+ "Fake 4-dimensional embedding vectors for each document"+ [[0.12 -0.34 0.56 -0.78]+ [0.23 0.45 -0.67 0.89]+ [0.11 -0.22 0.33 -0.44]+ [0.55 0.66 -0.77 0.88]+ [0.10 0.20 -0.30 0.40]+ [0.19 0.30 -0.41 0.52]+ [0.15 -0.25 0.35 -0.45]+ [0.31 0.42 -0.53 0.64]+ [0.75 -0.86 0.97 -0.18]+ [0.29 0.38 -0.47 0.56]+ [0.65 -0.74 0.83 -0.92]+ [0.77 -0.88 0.99 -0.19]+ [0.21 0.32 -0.43 0.54]+ [0.79 -0.89 0.91 -0.20]])++(defn- slightly-different-vector+ "Make a slightly different vector for the query so it's similar but not identical to the document."+ [v]+ (mapv #(+ % 0.01) v))+(def mock-embeddings- "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing. Each pair of strings represents a- document and a search query that should be most semantically similar to it, according to the embeddings."- {"Dog Training Guide" [0.12 -0.34 0.56 -0.78]- "puppy" [0.13 -0.33 0.57 -0.77]- "Bird Watching Tips" [0.23 0.45 -0.67 0.89]- "avian" [0.24 0.46 -0.66 0.88]- "Cat Behavior Study" [0.11 -0.22 0.33 -0.44]- "feline" [0.12 -0.21 0.34 -0.43]- "Horse Racing Analysis" [0.55 0.66 -0.77 0.88]- "equine" [0.56 0.67 -0.76 0.87]- "Fish Tank Setup" [0.10 0.20 -0.30 0.40]- "aquatic" [0.11 0.21 -0.29 0.39]- "Elephant Migration" [0.15 -0.25 0.35 -0.45]- "pachyderm" [0.16 -0.24 0.36 -0.44]- "Lion Pride Dynamics" [0.31 0.42 -0.53 0.64]- "predator" [0.32 0.43 -0.52 0.63]- "Penguin Colony Study" [0.75 -0.86 0.97 -0.18]- "Antarctic wildlife" [0.76 -0.85 0.96 -0.17]- "Whale Communication" [0.29 0.38 -0.47 0.56]- "marine mammal" [0.30 0.39 -0.46 0.55]- "Tiger Conservation" [0.65 -0.74 0.83 -0.92]- "endangered species" [0.66 -0.73 0.84 -0.91]- "Butterfly Migration" [0.17 0.28 -0.39 0.50]- "insect patterns" [0.18 0.29 -0.38 0.49]})+ "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing."+ (into {}+ (mapcat (fn [doc base-vec]+ (let [embeddable-text (#'search.ingestion/embeddable-text doc)+ query (:query doc)]+ [[embeddable-text base-vec]+ [query (slightly-different-vector base-vec)]]))+ mock-documents-for-embeddings+ base-embedding-vectors)))(defn get-mock-embedding"Lookup the embedding for `text` in [[mock-embeddings]]."@@ -270,6 +297,7 @@:id 123:name "Dog Training Guide":searchable_text "Dog Training Guide"+ :embeddable_text "Dog Training Guide":created_at #t "2025-01-01T12:00:00Z":creator_id 1:archived false@@ -284,6 +312,7 @@:id 456:name "Elephant Migration":searchable_text "Elephant Migration"+ :embeddable_text "Elephant Migration":created_at #t "2025-02-01T12:00:00Z":creator_id 2:archived true@@ -293,7 +322,8 @@(defn filter-for-mock-embeddings"Filter results to only include items whose names are keys in mock-embeddings map."[results]- (filter #(contains? mock-embeddings (:name %)) results))+ (let [mock-document-names (set (map :name mock-documents-for-embeddings))]+ (filter #(contains? mock-document-names (:name %)) results)))(defn closeable ^Closeable [o close-fn](reifydiff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 4e131d7e0af..3066b9c9990 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -63,6 +63,27 @@(->> (into [] xf search-terms)(str/join " "))))+(defn- embeddable-text+ "Generate labeled text for semantic search embeddings.+ Format:+ [model]+ field1: value1+ field2: value2++ Note: Unlike searchable-text, transformation functions in search-terms+ (e.g., explode-camel-case) are NOT applied. Transformations like camel-case+ explosion are specific to full-text search optimization."+ [m]+ (let [search-terms (:search-terms (search.spec/spec (:model m)))+ field-keys (cond-> search-terms (map? search-terms) keys)+ header (str "[" (:model m) "]")+ fields (keep (fn [k]+ (let [v (get m k)]+ (when (not (str/blank? (str v)))+ (str (name k) ": " (str/trim (str v))))))+ field-keys)]+ (str header "\n" (str/join "\n" fields))))+(defn- search-term-columns"Extract column names from search-terms spec for SQL query generation"[search-terms]@@ -107,7 +128,8 @@(assoc:display_data (display-data m):legacy_input (dissoc m :pinned :view_count :last_viewed_at :native_query)- :searchable_text (searchable-text m)))]+ :searchable_text (searchable-text m)+ :embeddable_text (embeddable-text m)))](merge fn-results sql-results)))(defn- attrs->select-items [attrs]diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 6bd98c3d27b..0b6cfe5b3b1 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -45,6 +45,46 @@(is (= "Test Name"(#'search.ingestion/searchable-text record)))))))+(deftest embeddable-text-test+ (testing "embeddable-text with vector format"+ (let [spec-fn (constantly {:search-terms [:name :description]})+ record {:model "card"+ :name "Sales Dashboard"+ :description "Shows quarterly sales data"+ :other-field "Other Value"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "[card]\nname: Sales Dashboard\ndescription: Shows quarterly sales data"+ (#'search.ingestion/embeddable-text record))))))++ (testing "embeddable-text with map format"+ (let [spec-fn (constantly {:search-terms {:name true+ :description true}})+ record {:model "dashboard"+ :name "Test Dashboard"+ :description "A test dashboard"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "[dashboard]\nname: Test Dashboard\ndescription: A test dashboard"+ (#'search.ingestion/embeddable-text record))))))++ (testing "embeddable-text filters out blank values"+ (let [spec-fn (constantly {:search-terms [:name :description :empty-field]})+ record {:model "card"+ :name "Test Card"+ :description " "+ :empty-field nil}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "[card]\nname: Test Card"+ (#'search.ingestion/embeddable-text record))))))++ (testing "embeddable-text does not apply transform functions"+ (let [spec-fn (constantly {:search-terms {:name search.spec/explode-camel-case}})+ record {:model "table"+ :name "CamelCaseTest"}]+ (with-redefs [search.spec/spec spec-fn]+ (is (= "[table]\nname: CamelCaseTest"+ (#'search.ingestion/embeddable-text record))+ "Transformation functions should not be applied to embeddable text for semantic search")))))+(deftest search-term-columns-test(testing "search-term-columns with vector format"(is (= #{:name :description}