Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[semantic search] Separate out embeddable_text from searchable_text and use labeled format for embedding (#64693)

Noah Mossb4c65c32993e
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index fba232ac253..dfec2274ead 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -118,7 +118,7 @@
(defn- doc->db-record
"Convert a document to a database record with a provided embedding."
- [embedding-vec {:keys [model id searchable_text created_at creator_id updated_at
+ [embedding-vec {:keys [model id searchable_text embeddable_text native_query created_at creator_id updated_at
last_editor_id archived verified official_collection database_id collection_id display_type legacy_input
pinned dashboardcard_count view_count last_viewed_at] :as doc}]
{:model model
@@ -128,7 +128,7 @@
:database_id database_id
:last_editor_id last_editor_id
:name (or (:name doc) "")
- :content searchable_text
+ :content embeddable_text
:display_type display_type
:archived (some-> archived to-boolean)
:official_collection (some-> official_collection to-boolean)
@@ -145,18 +145,18 @@
:text_search_vector (if (:name doc)
[:||
(search/weighted-tsvector "A" (:name doc))
- (search/weighted-tsvector "B" (or (:searchable_text doc) ""))]
- (search/weighted-tsvector "A" (or (:searchable_text doc) "")))
+ (search/weighted-tsvector "B" (or searchable_text ""))]
+ (search/weighted-tsvector "A" (or searchable_text "")))
:text_search_with_native_query_vector
(if (:name doc)
[:||
(search/weighted-tsvector "A" (:name doc))
(search/weighted-tsvector "B"
- (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
- (or (:native_query doc) "")])))]
+ (str/join " " (remove str/blank? [(or searchable_text "")
+ (or native_query "")])))]
(search/weighted-tsvector "A"
- (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
- (or (:native_query doc) "")]))))})
+ (str/join " " (remove str/blank? [(or searchable_text "")
+ (or native_query "")]))))})
(defn index-size
"Fetches the number of documents in the index table."
@@ -299,14 +299,14 @@
(defn- upsert-index-batch!
[connectable index documents & {:as opts}]
(when (seq documents)
- (let [text->docs (group-by :searchable_text documents)
- searchable-texts (keys text->docs)
+ (let [text->docs (group-by :embeddable_text documents)
+ embeddable-texts (keys text->docs)
upsert-embedding! (upsert-embedding!-fn connectable index text->docs)
[new-texts stats]
- (u/profile (str "Semantic search embedding caching attempt for " {:docs (count documents) :texts (count searchable-texts)})
- (let [[new-texts existing-embeddings] (partition-existing-embeddings connectable index searchable-texts)]
+ (u/profile (str "Semantic search embedding caching attempt for " {:docs (count documents) :texts (count embeddable-texts)})
+ (let [[new-texts existing-embeddings] (partition-existing-embeddings connectable index embeddable-texts)]
(if-not (seq existing-embeddings)
- [searchable-texts nil]
+ [embeddable-texts nil]
(u/profile (str "Semantic search cached embedding db update for " {:texts (count existing-embeddings)})
[new-texts (upsert-embedding! existing-embeddings)]))))]
(->>
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
index 7b7c5d9849f..2ebca765388 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
@@ -137,8 +137,8 @@
t1 (ts "2025-01-01T10:00:00Z")
t2 (ts "2025-01-01T11:00:00Z")
t3 (ts "2025-01-01T12:00:00Z")
- c1 {:model "card" :id "1" :name "Test" :searchable_text "Content"}
- c2 {:model "card" :id "2" :name "Test" :searchable_text "Content"}
+ c1 {:model "card" :id "1" :name "Test" :searchable_text "Content" :embeddable_text "Content"}
+ c2 {:model "card" :id "2" :name "Test" :searchable_text "Content" :embeddable_text "Content"}
version semantic.gate/search-doc->gate-doc
delete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))]
@@ -202,8 +202,8 @@
clock-ref (volatile! (Instant/parse "2025-01-04T00:00:00Z"))
clock (reify InstantSource (instant [_] @clock-ref))
t1 (ts "2025-01-01T10:00:00Z")
- c1 {:model "card" :id "1" :name "Test" :searchable_text "Content"}
- c2 {:model "card" :id "2" :name "Test" :searchable_text "Content"}
+ c1 {:model "card" :id "1" :name "Test" :searchable_text "Content" :embeddable_text "Content"}
+ c2 {:model "card" :id "2" :name "Test" :searchable_text "Content" :embeddable_text "Content"}
version semantic.gate/search-doc->gate-doc
add-gate-to-dlq! (fn [pgvector index-metadata index-id]
(->> (semantic.gate/poll pgvector index-metadata {})
@@ -309,7 +309,8 @@
(.setValue (json/encode {:model "card"
:name "hey"
:id "1"
- :searchable_text "foo"})))
+ :searchable_text "foo"
+ :embeddable_text "foo"})))
:gated_at (ts "2025-01-04T09:00:00Z")
:error_gated_at (ts "2025-01-04T09:00:00Z")}
{:id "card_2"
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
index 06b5aea0109..96ecb3a4694 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
@@ -37,6 +37,7 @@
search-doc {:model "card"
:id "123"
:searchable_text "Dog Training Guide"
+ :embeddable_text "Dog Training Guide"
:updated_at t1}
sut semantic.gate/search-doc->gate-doc]
(is (= {:id "card_123"
@@ -55,7 +56,7 @@
(:updated_at (sut (assoc search-doc :updated_at nil) t2))))))))
(deftest gate-doc->search-doc-test
- (let [original-search-doc {:model "card" :id "123" :searchable_text "Dog Training Guide"}
+ (let [original-search-doc {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}
gate-doc {:document (doto (PGobject.)
(.setType "jsonb")
(.setValue (json/encode original-search-doc)))}
@@ -96,10 +97,10 @@
t1 (ts "2025-01-01T00:01:00Z")
t2 (ts "2025-01-02T00:03:10Z")
t3 (ts "2025-01-03T00:02:42Z")
- c1 {:model "card" :id "123" :searchable_text "Dog Training Guide"}
- c2 {:model "card" :id "123" :searchable_text "Dog Training Guide 2"}
- c3 {:model "card" :id "123" :searchable_text "Dog Training Guide 3"}
- d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration"}
+ c1 {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}
+ c2 {:model "card" :id "123" :searchable_text "Dog Training Guide 2" :embeddable_text "Dog Training Guide 2"}
+ c3 {:model "card" :id "123" :searchable_text "Dog Training Guide 3" :embeddable_text "Dog Training Guide 3"}
+ d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration" :embeddable_text "Elephant Migration"}
version semantic.gate/search-doc->gate-doc
delete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))
sut semantic.gate/gate-documents!]
@@ -172,9 +173,9 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
epoch-watermark {:last-poll Instant/EPOCH :last-seen Instant/EPOCH}
- c1 {:model "card" :id "123" :searchable_text "a"}
- c2 {:model "card" :id "234" :searchable_text "b"}
- c3 {:model "card" :id "345" :searchable_text "c"}
+ c1 {:model "card" :id "123" :searchable_text "a" :embeddable_text "a"}
+ c2 {:model "card" :id "234" :searchable_text "b" :embeddable_text "b"}
+ c3 {:model "card" :id "345" :searchable_text "c" :embeddable_text "c"}
t0 (ts "2025-01-01T00:00:00Z")
t1 (ts "2025-01-01T00:01:00Z")
t2 (ts "2025-01-02T00:03:10Z")
@@ -334,8 +335,8 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
t1 (ts "2025-01-01T00:01:00Z")
- c1 {:model "card" :id "123" :searchable_text "Dog Training Guide"}
- d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration"}
+ c1 {:model "card" :id "123" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}
+ d1 {:model "dashboard" :id "456" :searchable_text "Elephant Migration" :embeddable_text "Elephant Migration"}
version semantic.gate/search-doc->gate-doc
sut semantic.gate/gate-documents!]
(with-open [_ (open-tables! pgvector index-metadata)]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 99bbdbde9d3..e8342ab8ba3 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -213,6 +213,7 @@
:id "1"
:name "Dog Training Guide"
:searchable_text "Dog Training Guide"
+ :embeddable_text "Dog Training Guide"
:creator_id 1
:legacy_input {:model "card" :id "1"}
:metadata {}}
@@ -220,6 +221,7 @@
:id "2"
:name "Elephant Migration"
:searchable_text "Elephant Migration"
+ :embeddable_text "Elephant Migration"
:creator_id 2
:legacy_input {:model "card" :id "2"}
:metadata {}}
@@ -227,6 +229,7 @@
:id "3"
:name "Tiger Conservation"
:searchable_text "Tiger Conservation"
+ :embeddable_text "Tiger Conservation"
:creator_id 3
:legacy_input {:model "card" :id "3"}
:metadata {}}]]
@@ -255,6 +258,7 @@
:id "1"
:name "Dog Training Guide"
:searchable_text "Dog Training Guide"
+ :embeddable_text "Dog Training Guide"
:creator_id 1
:legacy_input {:model "card" :id "1"}
:metadata {}}
@@ -262,6 +266,7 @@
:id "2"
:name "Elephant Migration"
:searchable_text "Elephant Migration"
+ :embeddable_text "Elephant Migration"
:creator_id 2
:legacy_input {:model "card" :id "2"}
:metadata {}}
@@ -269,6 +274,7 @@
:id "3"
:name "Dog Training Guide"
:searchable_text "Dog Training Guide"
+ :embeddable_text "Dog Training Guide"
:creator_id 3
:legacy_input {:model "card" :id "3"}
:metadata {}}]]
@@ -533,6 +539,7 @@
base-doc {:model "card"
:id "123"
:searchable_text "test content"
+ :embeddable_text "test content"
:creator_id 1}]
(testing "MySQL-style integer booleans are converted to real booleans"
@@ -587,6 +594,7 @@
:id (:id model-1)
:name (:name model-1)
:searchable_text (:name model-1)
+ :embeddable_text (:name model-1)
:created_at #t "2025-01-01T12:00:00Z"
:creator_id (mt/user->id :crowberto)
:archived false
@@ -601,6 +609,7 @@
:archived false
:collection_id coll-id
:searchable_text "Antarctic wildlife"
+ :embeddable_text "Antarctic wildlife"
:legacy_input {:id (str (:id model-index-1) ":1234")
:name "Antarctic wildlife"
:model "indexed-entity"}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj
index 176e714812d..335bef386f8 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj
@@ -41,9 +41,9 @@
t1 (ts "2025-01-01T00:01:00Z")
t2 (ts "2025-01-02T00:03:10Z")
t3 (ts "2025-01-03T00:02:42Z")
- c1 {:model "card" :id "1" :name "Poodle" :searchable_text "Dog Training Guide"}
- c2 {:model "card" :id "2" :name "Pug" :searchable_text "Dog Training Guide 2"}
- c3 {:model "card" :id "3" :name "Collie" :searchable_text "Dog Training Guide 3"}
+ c1 {:model "card" :id "1" :name "Poodle" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}
+ c2 {:model "card" :id "2" :name "Pug" :searchable_text "Dog Training Guide 2" :embeddable_text "Dog Training Guide 2"}
+ c3 {:model "card" :id "3" :name "Collie" :searchable_text "Dog Training Guide 3" :embeddable_text "Dog Training Guide 3"}
version semantic.gate/search-doc->gate-doc
delete (fn [doc t] (semantic.gate/deleted-search-doc->gate-doc (:model doc) (:id doc) t))]
(with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
@@ -289,7 +289,7 @@
model semantic.tu/mock-embedding-model
index (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
t1 (ts "2025-01-01T00:01:00Z")
- c1 {:model "card" :id "1" :name "Dog" :searchable_text "Dog Training Guide"}
+ c1 {:model "card" :id "1" :name "Dog" :searchable_text "Dog Training Guide" :embeddable_text "Dog Training Guide"}
version semantic.gate/search-doc->gate-doc]
(with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
_ (semantic.tu/open-index! pgvector index)]
@@ -549,7 +549,7 @@
clock-ref (volatile! (Instant/parse "2025-01-04T10:00:00Z"))
clock (reify InstantSource (instant [_] @clock-ref))
t1 (ts "2025-01-01T00:01:00Z")
- card (fn [id] {:model "card" :id (str id) :name "Test" :searchable_text "Content"})
+ card (fn [id] {:model "card" :id (str id) :name "Test" :searchable_text "Content" :embeddable_text "Content"})
version semantic.gate/search-doc->gate-doc
fresh-indexing-state (fn []
(let [state (semantic.indexer/init-indexing-state (get-metadata-row! pgvector index-metadata index))]
@@ -668,7 +668,7 @@
clock-ref (volatile! (Instant/parse "2025-01-04T10:00:00Z"))
clock (reify InstantSource (instant [_] @clock-ref))
t1 (ts "2025-01-01T00:01:00Z")
- card (fn [id content] {:model "card" :id (str id) :name content :searchable_text content})
+ card (fn [id content] {:model "card" :id (str id) :name content :searchable_text content :embeddable_text content})
version semantic.gate/search-doc->gate-doc
poisoned-doc-id (volatile! nil)
get-indexed-docs (fn []
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
index 5e89d435c50..dcec06566b7 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
@@ -173,7 +173,7 @@
remove-scores (fn [rows] (mapv #(dissoc % :score :all-scores) rows)) ; scores have time-sensitives components
sut* semantic.pgvector-api/query
sut #(remove-scores (:results (mt/as-admin (apply sut* %&)))) ; see notes below about perms
- search-string "insect patterns" ; specifics of search will be handled under index tests
+ search-string "puppy" ; specifics of search will be handled under index tests
_ (assert (semantic.tu/mock-embeddings search-string)
"search string should have test embedding")
search {:search-string search-string}]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
index 13285d4f37b..960a1160190 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
@@ -34,6 +34,7 @@
:id (str id)
:name name
:searchable_text name
+ :embeddable_text name
:updated_at (t/instant)
:creator_id 1})
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
index b5436bc6543..5a882bb8ba7 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
@@ -19,6 +19,7 @@
[doc]
(merge {:id 123
:searchable_text (:name doc)
+ :embeddable_text (:name doc)
:created_at #t "2025-01-01T12:00:00Z"
:creator_id (mt/user->id :rasta)
:archived false
@@ -70,9 +71,9 @@
(with-index-contents!
[{:model "card" :id 1 :name "orders"}
{:model "card" :id 2 :name "unrelated"}
- {:model "card" :id 3 :name "classified" :searchable_text "available only by court order"}
+ {:model "card" :id 3 :name "classified" :searchable_text "available only by court order" :embeddable_text "available only by court order"}
{:model "card" :id 4 :name "order"}
- {:model "card" :id 5 :name "orders, invoices, other stuff", :searchable_text "a verbose description"}
+ {:model "card" :id 5 :name "orders, invoices, other stuff", :searchable_text "a verbose description" :embeddable_text "a verbose description"}
{:model "card" :id 6 :name "ordering"}]
(with-redefs [semantic.tu/mock-embeddings {"order" [0.11 -0.33 0.56 -0.77]
"orders" [0.12 -0.34 0.57 -0.78]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index f93a5f61484..73fed24b551 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -175,31 +175,58 @@
`(with-weights {:rrf 1}
~@body))
+(def ^:private mock-documents-for-embeddings
+ "Mock documents and queries for testing, associated with fake embedding vectors in [[mock-embeddings]]."
+ [{:model "card" :name "Dog Training Guide" :query "puppy"}
+ {:model "card" :name "Bird Watching Tips" :query "avian"}
+ {:model "card" :name "Cat Behavior Study" :query "feline"}
+ {:model "card" :name "Horse Racing Analysis" :query "equine"}
+ {:model "card" :name "Fish Tank Setup" :query "aquatic"}
+ {:model "card" :name "Bigfoot Sightings" :query "spooky video evidence"}
+ {:model "dashboard" :name "Elephant Migration" :query "pachyderm"}
+ {:model "dashboard" :name "Lion Pride Dynamics" :query "predator"}
+ {:model "dashboard" :name "Penguin Colony Study" :query "Antarctic wildlife"}
+ {:model "dashboard" :name "Whale Communication" :query "marine mammal"}
+ {:model "dashboard" :name "Tiger Conservation" :query "endangered species"}
+ {:model "dashboard" :name "Loch Ness Stuff" :query "prehistoric monsters"}
+ ;; Tables include display_name in their search-terms (see table.clj:387-389), which gets
+ ;; included in embeddable_text as "display_name: Species Table". We must include it here
+ ;; so the mock embeddings match what gets indexed.
+ {:model "table" :name "Species Table" :display_name "Species Table" :query "taxonomy"}
+ {:model "table" :name "Monsters Table" :display_name "Monsters Table" :query "monster facts"}])
+
+(def ^:private base-embedding-vectors
+ "Fake 4-dimensional embedding vectors for each document"
+ [[0.12 -0.34 0.56 -0.78]
+ [0.23 0.45 -0.67 0.89]
+ [0.11 -0.22 0.33 -0.44]
+ [0.55 0.66 -0.77 0.88]
+ [0.10 0.20 -0.30 0.40]
+ [0.19 0.30 -0.41 0.52]
+ [0.15 -0.25 0.35 -0.45]
+ [0.31 0.42 -0.53 0.64]
+ [0.75 -0.86 0.97 -0.18]
+ [0.29 0.38 -0.47 0.56]
+ [0.65 -0.74 0.83 -0.92]
+ [0.77 -0.88 0.99 -0.19]
+ [0.21 0.32 -0.43 0.54]
+ [0.79 -0.89 0.91 -0.20]])
+
+(defn- slightly-different-vector
+ "Make a slightly different vector for the query so it's similar but not identical to the document."
+ [v]
+ (mapv #(+ % 0.01) v))
+
(def mock-embeddings
- "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing. Each pair of strings represents a
- document and a search query that should be most semantically similar to it, according to the embeddings."
- {"Dog Training Guide" [0.12 -0.34 0.56 -0.78]
- "puppy" [0.13 -0.33 0.57 -0.77]
- "Bird Watching Tips" [0.23 0.45 -0.67 0.89]
- "avian" [0.24 0.46 -0.66 0.88]
- "Cat Behavior Study" [0.11 -0.22 0.33 -0.44]
- "feline" [0.12 -0.21 0.34 -0.43]
- "Horse Racing Analysis" [0.55 0.66 -0.77 0.88]
- "equine" [0.56 0.67 -0.76 0.87]
- "Fish Tank Setup" [0.10 0.20 -0.30 0.40]
- "aquatic" [0.11 0.21 -0.29 0.39]
- "Elephant Migration" [0.15 -0.25 0.35 -0.45]
- "pachyderm" [0.16 -0.24 0.36 -0.44]
- "Lion Pride Dynamics" [0.31 0.42 -0.53 0.64]
- "predator" [0.32 0.43 -0.52 0.63]
- "Penguin Colony Study" [0.75 -0.86 0.97 -0.18]
- "Antarctic wildlife" [0.76 -0.85 0.96 -0.17]
- "Whale Communication" [0.29 0.38 -0.47 0.56]
- "marine mammal" [0.30 0.39 -0.46 0.55]
- "Tiger Conservation" [0.65 -0.74 0.83 -0.92]
- "endangered species" [0.66 -0.73 0.84 -0.91]
- "Butterfly Migration" [0.17 0.28 -0.39 0.50]
- "insect patterns" [0.18 0.29 -0.38 0.49]})
+ "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing."
+ (into {}
+ (mapcat (fn [doc base-vec]
+ (let [embeddable-text (#'search.ingestion/embeddable-text doc)
+ query (:query doc)]
+ [[embeddable-text base-vec]
+ [query (slightly-different-vector base-vec)]]))
+ mock-documents-for-embeddings
+ base-embedding-vectors)))
(defn get-mock-embedding
"Lookup the embedding for `text` in [[mock-embeddings]]."
@@ -270,6 +297,7 @@
:id 123
:name "Dog Training Guide"
:searchable_text "Dog Training Guide"
+ :embeddable_text "Dog Training Guide"
:created_at #t "2025-01-01T12:00:00Z"
:creator_id 1
:archived false
@@ -284,6 +312,7 @@
:id 456
:name "Elephant Migration"
:searchable_text "Elephant Migration"
+ :embeddable_text "Elephant Migration"
:created_at #t "2025-02-01T12:00:00Z"
:creator_id 2
:archived true
@@ -293,7 +322,8 @@
(defn filter-for-mock-embeddings
"Filter results to only include items whose names are keys in mock-embeddings map."
[results]
- (filter #(contains? mock-embeddings (:name %)) results))
+ (let [mock-document-names (set (map :name mock-documents-for-embeddings))]
+ (filter #(contains? mock-document-names (:name %)) results)))
(defn closeable ^Closeable [o close-fn]
(reify
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 4e131d7e0af..3066b9c9990 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -63,6 +63,27 @@
(->> (into [] xf search-terms)
(str/join " "))))
+(defn- embeddable-text
+ "Generate labeled text for semantic search embeddings.
+ Format:
+ [model]
+ field1: value1
+ field2: value2
+
+ Note: Unlike searchable-text, transformation functions in search-terms
+ (e.g., explode-camel-case) are NOT applied. Transformations like camel-case
+ explosion are specific to full-text search optimization."
+ [m]
+ (let [search-terms (:search-terms (search.spec/spec (:model m)))
+ field-keys (cond-> search-terms (map? search-terms) keys)
+ header (str "[" (:model m) "]")
+ fields (keep (fn [k]
+ (let [v (get m k)]
+ (when (not (str/blank? (str v)))
+ (str (name k) ": " (str/trim (str v))))))
+ field-keys)]
+ (str header "\n" (str/join "\n" fields))))
+
(defn- search-term-columns
"Extract column names from search-terms spec for SQL query generation"
[search-terms]
@@ -107,7 +128,8 @@
(assoc
:display_data (display-data m)
:legacy_input (dissoc m :pinned :view_count :last_viewed_at :native_query)
- :searchable_text (searchable-text m)))]
+ :searchable_text (searchable-text m)
+ :embeddable_text (embeddable-text m)))]
(merge fn-results sql-results)))
(defn- attrs->select-items [attrs]
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 6bd98c3d27b..0b6cfe5b3b1 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -45,6 +45,46 @@
(is (= "Test Name"
(#'search.ingestion/searchable-text record)))))))
+(deftest embeddable-text-test
+ (testing "embeddable-text with vector format"
+ (let [spec-fn (constantly {:search-terms [:name :description]})
+ record {:model "card"
+ :name "Sales Dashboard"
+ :description "Shows quarterly sales data"
+ :other-field "Other Value"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "[card]\nname: Sales Dashboard\ndescription: Shows quarterly sales data"
+ (#'search.ingestion/embeddable-text record))))))
+
+ (testing "embeddable-text with map format"
+ (let [spec-fn (constantly {:search-terms {:name true
+ :description true}})
+ record {:model "dashboard"
+ :name "Test Dashboard"
+ :description "A test dashboard"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "[dashboard]\nname: Test Dashboard\ndescription: A test dashboard"
+ (#'search.ingestion/embeddable-text record))))))
+
+ (testing "embeddable-text filters out blank values"
+ (let [spec-fn (constantly {:search-terms [:name :description :empty-field]})
+ record {:model "card"
+ :name "Test Card"
+ :description " "
+ :empty-field nil}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "[card]\nname: Test Card"
+ (#'search.ingestion/embeddable-text record))))))
+
+ (testing "embeddable-text does not apply transform functions"
+ (let [spec-fn (constantly {:search-terms {:name search.spec/explode-camel-case}})
+ record {:model "table"
+ :name "CamelCaseTest"}]
+ (with-redefs [search.spec/spec spec-fn]
+ (is (= "[table]\nname: CamelCaseTest"
+ (#'search.ingestion/embeddable-text record))
+ "Transformation functions should not be applied to embeddable text for semantic search")))))
+
(deftest search-term-columns-test
(testing "search-term-columns with vector format"
(is (= #{:name :description}