Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add search_curated tool for NLQ (#75237)

Chris Truter0bc758804b80
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 92743f522fb..58d115d4441 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -444,6 +444,7 @@
instaparse.core/defparser clojure.core/def
metabase-enterprise.action-v2.test-util/with-test-tables! clojure.core/let
metabase-enterprise.content-translation.routes-test/with-new-secret-key! clojure.core/fn
+ metabase-enterprise.curated-search.reconcile-test/with-isolated-mirror clojure.core/fn
metabase-enterprise.serialization.metadata-file-import.wire-format-test/with-tiny-fixture! clojure.core/fn
metabase-enterprise.serialization.test-util/with-dbs clojure.core/fn
metabase-enterprise.serialization.test-util/with-random-dump-dir clojure.core/let
@@ -460,6 +461,7 @@
metabase.channel.impl.http-test/with-captured-http-requests clojure.core/fn
metabase.channel.impl.http-test/with-server clojure.core/let
metabase.collections.models.collection-test/with-collection-in-location clojure.core/let
+ metabase.curated-search.api-test/with-test-entry clojure.core/let
metabase.driver.mongo.connection/with-mongo-client clojure.core/let
metabase.driver.mongo.connection/with-mongo-database clojure.core/let
metabase.driver.mongo.query-processor/mongo-let clojure.core/let
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 273ceace815..034e1335935 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -682,6 +682,18 @@
:api #{}
:uses :any}
+ curated-search
+ {:team "Metabot"
+ :api #{metabase.curated-search.api
+ metabase.curated-search.core}
+ :uses #{api
+ models
+ premium-features
+ request
+ util}
+ :model-exports #{:model/CuratedSearchEntry}
+ :model-imports #{:model/Card}}
+
dashboards
{:team "UX West"
:api #{metabase.dashboards.autoplace
@@ -1263,6 +1275,7 @@
channel
collections
config
+ curated-search
documents
driver
interestingness
@@ -2951,6 +2964,15 @@
:api #{metabase-enterprise.core.init}
:uses :any}
+ enterprise/curated-search
+ {:team "Metabot"
+ :api #{metabase-enterprise.curated-search.init}
+ :uses #{premium-features
+ task
+ util
+ enterprise/semantic-search}
+ :model-imports #{:model/CuratedSearchEntry}}
+
enterprise/custom-viz-plugin
{:team "Gadget"
:api #{metabase-enterprise.custom-viz-plugin.api
@@ -3394,6 +3416,8 @@
{:team "Metabot"
:api #{metabase-enterprise.semantic-search.api
metabase-enterprise.semantic-search.core
+ metabase-enterprise.semantic-search.db.datasource
+ metabase-enterprise.semantic-search.embedding
metabase-enterprise.semantic-search.init}
:uses #{activity-feed
analytics
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index 4eaa04f99f2..d0dbcda1688 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -27,7 +27,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -86,7 +86,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -159,7 +159,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -232,7 +232,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index c441cf49ea9..227749a7ad8 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -10,6 +10,7 @@
[metabase-enterprise.advanced-config.init]
[metabase-enterprise.audit-app.init]
[metabase-enterprise.cache.init]
+ [metabase-enterprise.curated-search.init]
[metabase-enterprise.custom-viz-plugin.init]
[metabase-enterprise.data-complexity-score.init]
[metabase-enterprise.database-replication.init]
diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/core.clj b/enterprise/backend/src/metabase_enterprise/curated_search/core.clj
new file mode 100644
index 00000000000..d8cdadb31b5
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/curated_search/core.clj
@@ -0,0 +1,120 @@
+(ns metabase-enterprise.curated-search.core
+ "Enterprise implementation of the curated search mirror: pgvector-backed similarity search over the
+ curated `curated_search_entries` table, and the write-path nudge that keeps the mirror fresh.
+
+ OSS shims live in [[metabase.curated-search.mirror]]; the background sync they nudge is the
+ [[metabase-enterprise.curated-search.task.sync]] Quartz job."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [honey.sql :as sql]
+ [honey.sql.helpers :as sql.helpers]
+ [metabase-enterprise.curated-search.index-table :as index-table]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase.premium-features.core :as premium-features :refer [defenterprise]]
+ [metabase.task.core :as task]
+ [metabase.util.json :as json]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs])
+ (:import
+ (java.sql SQLException)
+ (org.postgresql.util PGobject)))
+
+(set! *warn-on-reflection* true)
+
+(def sync-job-key
+ "Quartz job key of the background sync; [[request-sync!]] triggers it and
+ [[metabase-enterprise.curated-search.task.sync]] schedules it."
+ (jobs/key "metabase-enterprise.curated-search.sync.job"))
+
+(defn pgvector-configured?
+ "True when a pgvector store is configured (read once at boot from MB_PGVECTOR_DB_URL).
+ The sync task gates scheduling on this rather than [[available?]], so the periodic safety net exists
+ even when the semantic-search feature is turned on after startup (a common onboarding flow)."
+ []
+ (string? (not-empty semantic.db.datasource/db-url)))
+
+(defn available?
+ "Whether the curated search mirror can run right now: a pgvector store is configured and the license
+ includes semantic search.
+ The feature check can flip at runtime (token entered post-boot), so callers re-evaluate per use."
+ []
+ (and (pgvector-configured?)
+ (premium-features/has-feature? :semantic-search)))
+
+(defenterprise request-sync!
+ "Trigger the background sync job to reconcile the mirror soon.
+ Fire-and-forget: [[metabase.task.core/trigger-now!]] swallows scheduler errors, and the job's periodic
+ schedule covers anything a lost trigger misses."
+ :feature :semantic-search
+ []
+ (when (available?)
+ (task/trigger-now! sync-job-key)))
+
+;; Weighted scoring shaped like regular search (metabase.search.scoring): each factor contributes
+;; weight * score; similarity (1 - cosine distance) is primary, verified a flat 0/1 boost.
+(def ^:private similarity-weight 1.0)
+(def ^:private verified-weight 0.1)
+(def ^:private default-limit 10)
+
+(defn- scorer [nm score weight]
+ (let [score (double score) weight (double weight)]
+ {:name nm :score score :weight weight :contribution (* score weight)}))
+
+(defn- score
+ "Build a weighted-scorer breakdown for one row, shaped like regular search's scoring.
+ Returns a `:scores` vector of `{:name :score :weight :contribution}` plus the weighted-sum
+ `:total_score`."
+ [{:keys [distance verified]}]
+ (let [scores [(scorer :similarity (- 1.0 (double distance)) similarity-weight)
+ (scorer :verified (if verified 1.0 0.0) verified-weight)]]
+ {:scores scores
+ :total_score (reduce + (map :contribution scores))}))
+
+(defn- decode-entity [v]
+ (cond
+ (instance? PGobject v) (json/decode (.getValue ^PGobject v) true)
+ (string? v) (json/decode v true)
+ :else v))
+
+(defenterprise search
+ "Find the saved prompts nearest to `user-search-prompt` by cosine distance, up to `limit`.
+ Results are ranked by blended score (similarity + verified boost), each shaped
+ `{:saved_search_prompt :usage_instructions :entity :score}`.
+ Returns [] when the pgvector store is unconfigured."
+ :feature :semantic-search
+ [user-search-prompt limit]
+ (if-not (available?)
+ []
+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
+ limit (or limit default-limit)
+ model (embedding/get-configured-model)
+ embedding (embedding/get-embedding model user-search-prompt
+ {:type :query :record-tokens? true})
+ lit (index-table/format-embedding embedding)
+ ;; Order by the blended score, not raw distance, so a verified row whose boost lifts it into
+ ;; the top N survives the LIMIT. No HNSW index could accelerate this expression — see
+ ;; index-table for why the table has none.
+ ranking (format "(embedding <=> %s) - (CASE WHEN verified THEN %s ELSE 0.0 END)"
+ lit verified-weight)
+ rows (try
+ (jdbc/execute!
+ pgvector
+ (-> (sql.helpers/select :search_prompt :usage_instructions :entity :verified
+ [[:raw (str "embedding <=> " lit)] :distance])
+ (sql.helpers/from (keyword index-table/*vectors-table*))
+ (sql.helpers/order-by [[:raw ranking] :asc])
+ (sql.helpers/limit limit)
+ (sql/format {:quoted true}))
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ ;; 42P01 = undefined table: the background sync hasn't created the mirror yet.
+ (catch SQLException e
+ (if (= "42P01" (.getSQLState e)) [] (throw e))))]
+ (->> rows
+ (map (fn [row]
+ {:saved_search_prompt (:search_prompt row)
+ :usage_instructions (:usage_instructions row)
+ :entity (decode-entity (:entity row))
+ :score (score row)}))
+ (sort-by (comp :total_score :score) >)
+ vec))))
diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/index_table.clj b/enterprise/backend/src/metabase_enterprise/curated_search/index_table.clj
new file mode 100644
index 00000000000..508c90c0a04
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/curated_search/index_table.clj
@@ -0,0 +1,144 @@
+(ns metabase-enterprise.curated-search.index-table
+ "Lifecycle of the pgvector tables backing the curated search mirror.
+
+ Two tables live in the pgvector store:
+
+ - the vectors table: one row per appdb `curated_search_entries` row, carrying the embedding, a copy of
+ the searchable fields, and a `content_hash` used by the reconciler to detect stale rows;
+ - a single-row meta table recording the embedding model identity and schema version the vectors table
+ was built for.
+
+ There is deliberately no HNSW index: the search path orders by the blended score (distance minus the
+ verified boost), which an HNSW index can't accelerate anyway, and an exact scan over a curated table
+ of this size is microseconds.
+
+ [[ensure-tables!]] is the only entry point: it idempotently creates both tables and, when the
+ configured embedding model or the schema version no longer matches the meta row, drops and recreates
+ the vectors table so the next reconcile re-embeds everything.
+ That drop-and-rebuild is the entire model-change story — the appdb table is authoritative and small,
+ so rebuilding from it is cheap."
+ (:require
+ [clojure.string :as str]
+ [honey.sql :as sql]
+ [honey.sql.helpers :as sql.helpers]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs])
+ (:import
+ (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *vectors-table*
+ "Vectors table name. Dynamic so tests can rebind it to an isolated table."
+ "curated_search_index")
+
+(def ^:dynamic *meta-table*
+ "Meta table name. Dynamic so tests can rebind it to an isolated table."
+ "curated_search_index_meta")
+
+(def schema-version
+ "Version of the vectors table schema.
+ Bump it to force a drop-and-rebuild on instances whose meta row records an older version."
+ ;; 2: dropped the HNSW index — the rebuild sheds it from tables built under version 1.
+ 2)
+
+;; Advisory lock serializing concurrent ensure-tables! calls (e.g. several cluster nodes starting at
+;; once). Arbitrary app-wide-unique constant; semantic-search's migration lock uses 19991.
+(def ^:private ensure-lock-id 20011)
+
+(defn- sql-format-quoted [honey-sql]
+ (sql/format honey-sql {:quoted true}))
+
+(defn format-embedding
+ "Format a float-array embedding as a pgvector SQL literal.
+ Throws if any element is not a finite number."
+ [embedding]
+ (doseq [v embedding]
+ ;; NaN/Infinity satisfy `number?` but render as SQL literals pgvector chokes on, so require finite.
+ (when-not (and (number? v) (Double/isFinite (double v)))
+ (throw (ex-info "Embedding contains invalid value" {:invalid-value v}))))
+ (str "'[" (str/join ", " embedding) "]'::vector"))
+
+(defn- create-meta-table-sql []
+ (-> (sql.helpers/create-table (keyword *meta-table*) :if-not-exists)
+ (sql.helpers/with-columns
+ [[:id :smallint [:primary-key] [:default 1] [:check [:= :id 1]]]
+ [:provider :text :not-null]
+ [:model_name :text :not-null]
+ [:vector_dimensions :int :not-null]
+ [:schema_version :int :not-null]
+ [:updated_at :timestamp-with-time-zone :not-null]])
+ sql-format-quoted))
+
+(defn- create-vectors-table-sql [dims]
+ (-> (sql.helpers/create-table (keyword *vectors-table*) :if-not-exists)
+ (sql.helpers/with-columns
+ [[:index_id :bigint [:primary-key]]
+ [:search_prompt :text :not-null]
+ [:usage_instructions :text :not-null]
+ [:entity :jsonb :not-null]
+ [:verified :boolean [:default false] :not-null]
+ [:content_hash :text :not-null]
+ [:embedding [:raw (format "vector(%d)" dims)] :not-null]])
+ sql-format-quoted))
+
+(defn- model-identity
+ "The part of the meta row that identifies what the vectors table was built for."
+ [embedding-model]
+ {:provider (:provider embedding-model)
+ :model_name (:model-name embedding-model)
+ :vector_dimensions (:vector-dimensions embedding-model)
+ :schema_version schema-version})
+
+(defn- read-meta [tx]
+ (jdbc/execute-one! tx [(format "SELECT provider, model_name, vector_dimensions, schema_version FROM \"%s\" WHERE id = 1" *meta-table*)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+
+(defn- write-meta! [tx embedding-model]
+ (let [{:keys [provider model_name vector_dimensions schema_version]} (model-identity embedding-model)]
+ (jdbc/execute! tx
+ (-> (sql.helpers/insert-into (keyword *meta-table*))
+ (sql.helpers/values [{:id 1
+ :provider provider
+ :model_name model_name
+ :vector_dimensions vector_dimensions
+ :schema_version schema_version
+ :updated_at (Instant/now)}])
+ (sql.helpers/on-conflict :id)
+ (sql.helpers/do-update-set :provider :model_name :vector_dimensions
+ :schema_version :updated_at)
+ sql-format-quoted))))
+
+(defn- create-tables! [tx dims]
+ (jdbc/execute! tx (create-vectors-table-sql dims)))
+
+(defn ensure-tables!
+ "Idempotently create the vectors + meta tables for `embedding-model`, rebuilding on model change.
+
+ When the meta row's model identity (provider, model name, dimensions) or schema version no longer
+ matches, the vectors table is dropped and recreated empty — the next reconcile re-embeds every row.
+ Serialized across nodes with an advisory lock. Returns :created, :rebuilt or :ok."
+ [pgvector embedding-model]
+ (jdbc/with-transaction [tx pgvector]
+ (jdbc/execute! tx [(format "SELECT pg_advisory_xact_lock(%d)" ensure-lock-id)])
+ (jdbc/execute! tx (sql/format (sql.helpers/create-extension :vector :if-not-exists)))
+ (jdbc/execute! tx (create-meta-table-sql))
+ (let [stored (read-meta tx)
+ current (model-identity embedding-model)
+ dims (:vector_dimensions current)]
+ (cond
+ (nil? stored)
+ (do (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :created)
+
+ (not= stored current)
+ (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS \"%s\"" *vectors-table*)])
+ (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :rebuilt)
+
+ :else
+ ;; Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table heals itself.
+ (do (create-tables! tx dims)
+ :ok)))))
diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/init.clj b/enterprise/backend/src/metabase_enterprise/curated_search/init.clj
new file mode 100644
index 00000000000..17174e85500
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/curated_search/init.clj
@@ -0,0 +1,5 @@
+(ns metabase-enterprise.curated-search.init
+ "Loads the curated-search module's side-effecting namespaces (the background sync task) at
+ system startup. See [[metabase-enterprise.core.init]]."
+ (:require
+ [metabase-enterprise.curated-search.task.sync]))
diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.clj b/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.clj
new file mode 100644
index 00000000000..c85f4404eaa
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.clj
@@ -0,0 +1,112 @@
+(ns metabase-enterprise.curated-search.reconcile
+ "Reconciles the pgvector mirror with the authoritative appdb `curated_search_entries` table.
+
+ Rather than streaming individual writes (the table is curated and small — hundreds to low thousands of
+ rows), [[reconcile!]] diffs the whole table against the mirror by content hash on every run: rows whose
+ hash is missing or stale are re-embedded and upserted, mirror rows with no appdb counterpart are
+ deleted, unchanged rows cost nothing.
+ Because each run re-derives the mirror from the appdb, any missed write — pgvector downtime, a crashed
+ node, an import that bypassed the model hooks — is repaired on the next run with no operator action.
+
+ Runs from the [[metabase-enterprise.curated-search.task.sync]] Quartz job; callers supply the
+ datasource and embedding model so this namespace reads no settings itself."
+ (:require
+ [buddy.core.hash :as buddy-hash]
+ [honey.sql :as sql]
+ [honey.sql.helpers :as sql.helpers]
+ [metabase-enterprise.curated-search.index-table :as index-table]
+ [metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private embed-batch-size
+ "Rows embedded per embedding-service request while backfilling stale mirror rows."
+ 50)
+
+(defn content-hash
+ "Hash of an appdb row's mirror-relevant fields.
+ Stored on the mirror row at upsert time; a mismatch (or absence) marks the row stale."
+ [{:keys [search_prompt usage_instructions entity verified]}]
+ (u/encode-base64-bytes
+ (buddy-hash/sha1
+ (json/encode (sorted-map :search_prompt search_prompt
+ :usage_instructions (or usage_instructions "")
+ :entity entity
+ :verified (boolean verified))))))
+
+(defn- mirror-hashes
+ "Map of index_id -> content_hash for every row currently in the mirror."
+ [pgvector]
+ (into {}
+ (map (juxt :index_id :content_hash))
+ (jdbc/execute! pgvector
+ [(format "SELECT index_id, content_hash FROM \"%s\"" index-table/*vectors-table*)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
+(defn- delete-rows! [pgvector ids]
+ (when (seq ids)
+ (jdbc/execute! pgvector
+ (-> (sql.helpers/delete-from (keyword index-table/*vectors-table*))
+ (sql.helpers/where [:in :index_id ids])
+ (sql/format {:quoted true})))))
+
+(defn- row->record [row hash embedding]
+ {:index_id (:id row)
+ :search_prompt (:search_prompt row)
+ :usage_instructions (or (:usage_instructions row) "")
+ :entity [:cast (json/encode (:entity row)) :jsonb]
+ :verified (boolean (:verified row))
+ :content_hash hash
+ :embedding [:raw (index-table/format-embedding embedding)]})
+
+(defn- upsert-batch!
+ "Embed one batch of stale appdb rows and upsert them into the mirror.
+ Returns the number of rows upserted."
+ [pgvector embedding-model rows]
+ (let [embeddings (embedding/get-embeddings-batch embedding-model (map :search_prompt rows)
+ {:type :index :record-tokens? true})
+ records (map (fn [row embedding] (row->record row (content-hash row) embedding))
+ rows embeddings)]
+ (jdbc/execute! pgvector
+ (-> (sql.helpers/insert-into (keyword index-table/*vectors-table*))
+ (sql.helpers/values (vec records))
+ (sql.helpers/on-conflict :index_id)
+ (sql.helpers/do-update-set :search_prompt :usage_instructions :entity
+ :verified :content_hash :embedding)
+ (sql/format {:quoted true})))
+ (count rows)))
+
+(defn reconcile!
+ "Bring the pgvector mirror in line with the appdb table; see the namespace docstring for the approach.
+ Embedding-service calls are scoped to changed rows only.
+ A failed batch is logged and skipped — its rows stay stale and are retried on the next run.
+ Returns {:upserted n :deleted n :unchanged n}."
+ [pgvector embedding-model]
+ (index-table/ensure-tables! pgvector embedding-model)
+ (let [appdb-rows (t2/select [:model/CuratedSearchEntry :id :search_prompt :usage_instructions
+ :entity :verified])
+ mirrored (mirror-hashes pgvector)
+ stale (remove #(= (content-hash %) (get mirrored (:id %))) appdb-rows)
+ orphans (remove (set (map :id appdb-rows)) (keys mirrored))
+ upserted (transduce
+ (partition-all embed-batch-size)
+ (completing
+ (fn [n batch]
+ (+ n (try
+ (upsert-batch! pgvector embedding-model batch)
+ (catch Exception e
+ (log/error e "curated search mirror: failed to upsert batch of"
+ (count batch) "rows; will retry next run")
+ 0)))))
+ 0
+ stale)]
+ (delete-rows! pgvector orphans)
+ {:upserted upserted
+ :deleted (count orphans)
+ :unchanged (- (count appdb-rows) (count stale))}))
diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.clj b/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.clj
new file mode 100644
index 00000000000..2829ff740c1
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.clj
@@ -0,0 +1,59 @@
+(ns metabase-enterprise.curated-search.task.sync
+ "Quartz job running the curated search mirror reconciliation.
+
+ Scheduled periodically as a safety net; appdb writes also trigger it immediately via
+ [[metabase-enterprise.curated-search.core/request-sync!]], so curator edits become searchable
+ within seconds.
+ `DisallowConcurrentExecution` plus Quartz's trigger coalescing debounce edit bursts to at most one
+ queued extra run, and Quartz clustering ensures a single node runs it."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.simple :as simple]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase-enterprise.curated-search.core :as curated-search.core]
+ [metabase-enterprise.curated-search.reconcile :as reconcile]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log])
+ (:import
+ (java.time Duration Instant)
+ (java.util Date)))
+
+(set! *warn-on-reflection* true)
+
+(task/defjob ^{org.quartz.DisallowConcurrentExecution true
+ :doc "Reconciles the curated search pgvector mirror with the appdb table."}
+ CuratedSearchEntrySync [_ctx]
+ (when (curated-search.core/available?)
+ (try
+ (let [result (reconcile/reconcile! (semantic.db.datasource/ensure-initialized-data-source!)
+ (embedding/get-configured-model))]
+ (when (pos? (+ (:upserted result) (:deleted result)))
+ (log/info "curated search mirror reconciled" result)))
+ (catch Throwable e
+ ;; Log and move on: the next periodic run retries from the authoritative appdb table.
+ (log/error e "curated search mirror reconciliation failed")))))
+
+(def ^:private ^Duration startup-delay (Duration/parse "PT10S"))
+(def ^:private ^Duration run-frequency (Duration/parse "PT30S"))
+
+(defmethod task/init! ::CuratedSearchEntrySync [_]
+ ;; Gate scheduling on pgvector being configured (boot-fixed), NOT on available? — the job body
+ ;; self-gates on the feature flag, so scheduling here lets the periodic safety net (and the
+ ;; write-path trigger's target job) exist even when the license is enabled after startup.
+ (when (curated-search.core/pgvector-configured?)
+ (let [job (jobs/build
+ (jobs/of-type CuratedSearchEntrySync)
+ (jobs/store-durably)
+ (jobs/with-identity curated-search.core/sync-job-key))
+ trigger (triggers/build
+ (triggers/with-identity
+ (triggers/key "metabase-enterprise.curated-search.sync.trigger"))
+ (triggers/for-job curated-search.core/sync-job-key)
+ (triggers/start-at (Date/from (.plus (Instant/now) startup-delay)))
+ (triggers/with-schedule
+ (simple/schedule
+ (simple/with-interval-in-milliseconds (.toMillis run-frequency))
+ (simple/repeat-forever))))]
+ (task/schedule-task! job trigger))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 700fcdd57a0..120114582d6 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -42,7 +42,10 @@
(conj "EmbeddingTheme")
(not (:no-custom-viz-plugins opts))
- (conj "CustomVizPlugin")))
+ (conj "CustomVizPlugin")
+
+ (not (:no-curated-search opts))
+ (conj "CuratedSearchEntry")))
(defn make-targets-of-type
"Returns a targets seq with model type and given ids"
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index 8a41f1bd9c7..c90393d4394 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -82,7 +82,7 @@
(def legal-top-level-paths
"Known top-level paths for directory with serialization output.
We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."
- #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "snippets" "transforms"})
+ #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "curated_search_entries" "snippets" "transforms"})
(defn- ingest-all
"Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 17aa985bb26..d18257ffbd3 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -33,7 +33,8 @@
"The list of all models exported by serialization by default. Used for production code and by tests."
(concat data-model
content
- ["CustomVizPlugin"
+ ["CuratedSearchEntry"
+ "CustomVizPlugin"
"EmbeddingTheme"
"FieldValues"
"Metabot"
diff --git a/enterprise/backend/test/metabase_enterprise/curated_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/curated_search/core_test.clj
new file mode 100644
index 00000000000..2138f07a394
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/curated_search/core_test.clj
@@ -0,0 +1,150 @@
+(ns metabase-enterprise.curated-search.core-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.curated-search.core :as curated-search.core]
+ [metabase-enterprise.curated-search.index-table :as index-table]
+ [metabase-enterprise.curated-search.reconcile :as reconcile]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.curated-search.mirror :as mirror]
+ [metabase.metabot.tools.curated-search :as tools.curated-search]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]))
+
+(set! *warn-on-reflection* true)
+
+(defn- approx [target] #(< (abs (- (double %) (double target))) 1e-9))
+
+(deftest score-shape-matches-regular-search-test
+ (let [score (var-get #'curated-search.core/score)]
+ (testing "weighted-scorer breakdown: per-factor {name score weight contribution} + total_score"
+ (is (=? {:scores [{:name :similarity :score (approx 0.8) :weight 1.0 :contribution (approx 0.8)}
+ {:name :verified :score 0.0 :weight 0.1 :contribution 0.0}]
+ :total_score (approx 0.8)}
+ (score {:distance 0.2 :verified false}))))
+ (testing "the verified boost applies its weight and strictly increases the total"
+ (is (=? {:total_score (approx 0.9)} (score {:distance 0.2 :verified true})))
+ (let [s (score {:distance 0.2 :verified true})]
+ (is (=? (approx (reduce + (map :contribution (:scores s)))) (:total_score s))))
+ (is (> (:total_score (score {:distance 0.2 :verified true}))
+ (:total_score (score {:distance 0.2 :verified false})))))))
+
+(deftest dispatch-without-pgvector-test
+ (testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"
+ ;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:
+ ;; available? is false, so search returns [] and the sync nudge no-ops rather than throwing.
+ (mt/with-premium-features #{:semantic-search}
+ (with-redefs [semantic.db.datasource/db-url nil]
+ (is (= [] (mirror/search "anything" 10)))
+ (is (nil? (mirror/request-sync!)))))))
+
+(deftest request-sync-triggers-the-job-test
+ (testing "when the mirror is available, the nudge fires trigger-now! on the sync job — and nothing else"
+ (mt/with-premium-features #{:semantic-search}
+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
+ (let [triggered (atom [])]
+ (mt/with-dynamic-fn-redefs [task/trigger-now! (fn [job-key] (swap! triggered conj job-key))]
+ (mirror/request-sync!)
+ (is (= [curated-search.core/sync-job-key] @triggered))))))))
+
+(deftest pgvector-configured-decoupled-from-feature-test
+ (testing "scheduling gates on pgvector config, independent of the feature flag, so a license enabled after boot still gets the periodic sync"
+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
+ (mt/with-premium-features #{}
+ (is (true? (curated-search.core/pgvector-configured?)))
+ (is (false? (curated-search.core/available?))))
+ (mt/with-premium-features #{:semantic-search}
+ (is (true? (curated-search.core/available?)))))))
+
+(deftest ^:sequential verified-boost-applies-before-limit-test
+ (testing "the SQL LIMIT ranks by blended score: a verified row slightly farther by raw distance still wins top-1"
+ (when semantic.db.datasource/db-url
+ (let [suffix (System/nanoTime)
+ ds (semantic.db.datasource/ensure-initialized-data-source!)
+ p-verif "verified entry"
+ p-plain "plain entry"
+ q "the query"]
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly semantic.tu/mock-embedding-model)]
+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix)
+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix)]
+ ;; plain ties the query exactly (distance 0); verified sits at distance ~0.006, within its
+ ;; 0.1 boost. A raw-distance LIMIT 1 would keep only the plain row.
+ (semantic.tu/with-mock-embeddings {q [1.0 0.0 0.0 0.0]
+ p-plain [1.0 0.0 0.0 0.0]
+ p-verif [0.9 0.1 0.0 0.0]}
+ (mt/with-temp [:model/CuratedSearchEntry _
+ {:search_prompt p-plain :entity {:model "table" :id 1}}
+ :model/CuratedSearchEntry _
+ {:search_prompt p-verif :verified true :entity {:model "table" :id 2}}]
+ (try
+ (reconcile/reconcile! ds semantic.tu/mock-embedding-model)
+ (is (= [p-verif] (mapv :saved_search_prompt (mirror/search q 1))))
+ (finally
+ (jdbc/execute! ds [(str "DROP TABLE IF EXISTS "
+ index-table/*vectors-table* ", "
+ index-table/*meta-table*)]))))))))))))
+
+(defn- create-entry!
+ "POST an entry through the CRUD API; returns the created row's id."
+ [search-prompt entity verified]
+ (:id (mt/user-http-request :crowberto :post 200 "curated-search/"
+ {:search_prompt search-prompt :entity entity :verified verified})))
+
+(deftest ^:sequential crud-api-to-tool-end-to-end-test
+ (testing "CRUD API write -> reconcile -> pgvector -> search_curated tool, end to end"
+ ;; Self-gated on MB_PGVECTOR_DB_URL — CI without semantic-search infra skips this; locally with the
+ ;; dev pgvector running it exercises the whole pipeline. Uses the mock embedding model (4-dim,
+ ;; deterministic, no network) against isolated temp tables, with the reconciler run directly in
+ ;; place of the background Quartz job.
+ (when semantic.db.datasource/db-url
+ (let [suffix (System/nanoTime)
+ ds (semantic.db.datasource/ensure-initialized-data-source!)
+ ent-1 {:model "table" :id (mt/id :orders)}
+ ent-2 {:model "table" :id (mt/id :people)}
+ ;; Both prompts + the query embed to the same vector, so cosine distance ties and the
+ ;; verified boost (0.10) alone decides the ranking.
+ vec1 [1.0 0.0 0.0 0.0]
+ p-verif "revenue by region (verified)"
+ p-plain "revenue by region (plain)"
+ q "monthly revenue per region"
+ ;; bind a user: the tool's hydration permission-filters via mi/can-read?, which needs
+ ;; *current-user-id* (a superuser can read the test tables).
+ search! #(mt/with-test-user :crowberto
+ (get-in (tools.curated-search/curated-search-tool
+ {:user_search_prompt q})
+ [:structured-output :data]))]
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly semantic.tu/mock-embedding-model)]
+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix)
+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix)]
+ (semantic.tu/with-mock-embeddings {p-verif vec1 p-plain vec1 q vec1}
+ (try
+ (let [id-verif (create-entry! p-verif ent-1 true)
+ id-plain (create-entry! p-plain ent-2 false)
+ _ (reconcile/reconcile! ds semantic.tu/mock-embedding-model)
+ results (search!)]
+ (testing "both distinct entities mirrored and returned, the verified one boosted first"
+ (is (= [p-verif p-plain] (mapv :saved_search_prompt results))))
+ (testing "data is per-entity hydrated records; total_score reflects the boost, similarity=1.0 (vectors tie)"
+ (is (=? [{:type "table" :id (mt/id :orders) :similarity (approx 1.0)
+ :score {:total_score (approx 1.1)}}
+ {:type "table" :id (mt/id :people) :similarity (approx 1.0)
+ :score {:total_score (approx 1.0)}}]
+ results)))
+ (testing "deleting via the CRUD API + reconcile removes the row from search results"
+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" id-verif))
+ (is (=? {:deleted 1} (reconcile/reconcile! ds semantic.tu/mock-embedding-model)))
+ (is (= [p-plain] (mapv :saved_search_prompt (search!)))))
+ ;; clean up the appdb rows we created
+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" id-plain)))
+ (finally
+ (jdbc/execute! ds [(str "DROP TABLE IF EXISTS "
+ index-table/*vectors-table* ", "
+ index-table/*meta-table*)])))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.clj
new file mode 100644
index 00000000000..a4537b6291a
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.clj
@@ -0,0 +1,119 @@
+(ns metabase-enterprise.curated-search.reconcile-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.curated-search.index-table :as index-table]
+ [metabase-enterprise.curated-search.reconcile :as reconcile]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest content-hash-test
+ (let [row {:search_prompt "p" :usage_instructions "u" :entity {:model "table" :id 1}
+ :verified false}]
+ (testing "equal rows hash equal; nil and absent usage_instructions hash like empty"
+ (is (= (reconcile/content-hash row) (reconcile/content-hash row)))
+ (is (= (reconcile/content-hash (assoc row :usage_instructions nil))
+ (reconcile/content-hash (assoc row :usage_instructions "")))))
+ (testing "every mirror-relevant field changes the hash"
+ (let [h (reconcile/content-hash row)]
+ (doseq [variant [(assoc row :search_prompt "p2")
+ (assoc row :usage_instructions "u2")
+ (assoc row :entity {:model "table" :id 2})
+ (assoc row :verified true)]]
+ (is (not= h (reconcile/content-hash variant)) (pr-str variant)))))))
+
+(deftest format-embedding-rejects-invalid-values-test
+ (testing "non-numbers, NaN and infinities are rejected before they reach a raw SQL literal"
+ (doseq [bad [Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY "0.1"]]
+ (is (thrown-with-msg? Exception #"invalid value"
+ (index-table/format-embedding [0.1 bad 0.3]))
+ (pr-str bad)))
+ (is (string? (index-table/format-embedding [0.1 -0.2 0.3])))))
+
+(defmacro ^:private with-isolated-mirror
+ "Run `body` with isolated pgvector tables bound and dropped afterwards, binding `ds-sym` to the
+ pgvector datasource. Skips the body entirely when no pgvector store is configured."
+ [[ds-sym] & body]
+ `(when semantic.db.datasource/db-url
+ (let [suffix# (System/nanoTime)
+ ~ds-sym (semantic.db.datasource/ensure-initialized-data-source!)]
+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix#)
+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix#)]
+ (try
+ ~@body
+ (finally
+ (jdbc/execute! ~ds-sym [(str "DROP TABLE IF EXISTS "
+ index-table/*vectors-table* ", "
+ index-table/*meta-table*)])))))))
+
+(defn- mirror-rows [ds]
+ (jdbc/execute! ds
+ [(format "SELECT index_id, search_prompt, verified, content_hash FROM \"%s\" ORDER BY index_id"
+ index-table/*vectors-table*)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+
+(deftest ^:sequential reconcile-lifecycle-test
+ (with-isolated-mirror [ds]
+ (let [model semantic.tu/mock-embedding-model]
+ (mt/with-temp [:model/CuratedSearchEntry {id-1 :id}
+ {:search_prompt "monthly revenue"
+ :entity {:model "table" :id 1}}
+ :model/CuratedSearchEntry {id-2 :id}
+ {:search_prompt "orders and customers" :verified true
+ :entity {:model "table" :id 2}}]
+ (let [appdb-total (t2/count :model/CuratedSearchEntry)]
+ (testing "first run mirrors every appdb row"
+ (is (=? {:upserted appdb-total :deleted 0 :unchanged 0}
+ (reconcile/reconcile! ds model)))
+ (is (=? {id-1 {:search_prompt "monthly revenue" :verified false}
+ id-2 {:search_prompt "orders and customers" :verified true}}
+ (-> (into {} (map (juxt :index_id identity)) (mirror-rows ds))
+ (select-keys [id-1 id-2])))))
+ (testing "an unchanged second run embeds and writes nothing"
+ (is (=? {:upserted 0 :deleted 0 :unchanged appdb-total}
+ (reconcile/reconcile! ds model))))
+ (testing "an updated row is re-embedded; the rest stay unchanged"
+ (t2/update! :model/CuratedSearchEntry id-1 {:verified true})
+ (is (=? {:upserted 1 :deleted 0 :unchanged (dec appdb-total)}
+ (reconcile/reconcile! ds model)))
+ (is (=? {:verified true}
+ (->> (mirror-rows ds) (filter #(= id-1 (:index_id %))) first))))
+ (testing "a write that bypassed the hooks entirely is still repaired (self-healing)"
+ ;; Tamper with the mirror behind the reconciler's back.
+ (jdbc/execute! ds [(format "UPDATE \"%s\" SET content_hash = 'bogus' WHERE index_id = %d"
+ index-table/*vectors-table* id-2)])
+ (is (=? {:upserted 1 :deleted 0}
+ (reconcile/reconcile! ds model))))))
+ (testing "rows deleted from the appdb are removed from the mirror as orphans"
+ ;; The with-temp rows above are gone now.
+ (is (=? {:upserted 0 :deleted 2}
+ (reconcile/reconcile! ds model)))))))
+
+(deftest ^:sequential rebuild-on-model-change-test
+ (with-isolated-mirror [ds]
+ (let [model semantic.tu/mock-embedding-model]
+ (mt/with-temp [:model/CuratedSearchEntry _
+ {:search_prompt "monthly revenue"
+ :entity {:model "table" :id 1}}]
+ (let [appdb-total (t2/count :model/CuratedSearchEntry)]
+ (testing "populate the mirror under the original model"
+ (is (=? {:upserted appdb-total} (reconcile/reconcile! ds model))))
+ (testing "a model-identity change drops the vectors table and the next run re-embeds everything"
+ (let [new-model (assoc model :model-name "model-v2")]
+ (is (= :rebuilt (index-table/ensure-tables! ds new-model)))
+ (is (= [] (mirror-rows ds)))
+ (is (=? {:upserted appdb-total :deleted 0 :unchanged 0}
+ (reconcile/reconcile! ds new-model)))
+ (testing "a schema-version mismatch alone also triggers the rebuild"
+ ;; stale-write the meta row, as if the table were built by an older code version
+ (jdbc/execute! ds [(format "UPDATE \"%s\" SET schema_version = schema_version - 1"
+ index-table/*meta-table*)])
+ (is (= :rebuilt (index-table/ensure-tables! ds new-model)))
+ (is (= [] (mirror-rows ds))))
+ (testing "the rebuild heals the meta row, so it doesn't recur on the next sync"
+ (is (= :ok (index-table/ensure-tables! ds new-model)))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.clj
new file mode 100644
index 00000000000..c80fa683851
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.clj
@@ -0,0 +1,136 @@
+(ns metabase-enterprise.serialization.v2.curated-search-entry-test
+ "Round-trip serialization tests for `:model/CuratedSearchEntry`, whose `entity` column holds a
+ polymorphic Card/Table reference that must survive export and import."
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.test-util :as ts]
+ [metabase-enterprise.serialization.v2.extract :as extract]
+ [metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
+ [metabase-enterprise.serialization.v2.load :as serdes.load]
+ [metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]
+ [metabase.models.serialization :as serdes]
+ [metabase.search.core :as search]
+ [metabase.test :as mt]
+ [metabase.warehouses.models.database :as models.database]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; The references point at the H2 test-data Table, so keep the H2 guard off and re-enable H2 in extract.
+#_{:clj-kondo/ignore [:metabase/validate-deftest]}
+(use-fixtures :each (fn [thunk]
+ (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
+ models.database/assert-not-h2! (constantly nil)]
+ (binding [models.database/*include-h2-in-extract?* true]
+ (thunk)))))
+
+(defn- no-labels [path]
+ (mapv #(dissoc % :label) path))
+
+(defn- ingestion-in-memory [extractions]
+ (let [mapped (into {} (for [entity (vec extractions)]
+ [(no-labels (serdes/path entity)) entity]))]
+ (reify serdes.ingest/Ingestable
+ (ingest-list [_] (keys mapped))
+ (ingest-one [_ path] (get mapped (no-labels path)))
+ (ingest-errors [_] []))))
+
+(deftest card-backed-entity-round-trip-test
+ (testing "every card-backed entity model exports as the Card's entity_id and imports back"
+ (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]
+ (doseq [entity-model ["card" "model" "metric" "question"]]
+ (testing entity-model
+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id sli-eid :entity_id}
+ {:search_prompt (str "find " entity-model) :verified true
+ :usage_instructions "Use it directly."
+ :entity {:model entity-model :id card-id}}]
+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)]
+ (testing "the entity ref is exported as a portable reference"
+ (is (=? {:entity_id sli-eid
+ :search_prompt (str "find " entity-model)
+ :usage_instructions "Use it directly."
+ :verified true
+ :entity {:model entity-model :id card-eid}}
+ extracted)))
+ (testing "dependencies cover the referenced Card"
+ (is (= #{[{:model "Card" :id card-eid}]}
+ (serdes/dependencies extracted))))
+ (testing "importing resolves the ref back to the local id"
+ (t2/delete! :model/CuratedSearchEntry :id sli-id)
+ (serdes.load/load-metabase! (ingestion-in-memory [extracted]))
+ (is (= {:model entity-model :id card-id}
+ (t2/select-one-fn :entity :model/CuratedSearchEntry :entity_id sli-eid)))
+ ;; load-metabase! re-inserts under a new id that with-temp can't reap; clean it up so the
+ ;; row doesn't leak into other tests' view of the (small, fully-scanned) table.
+ (t2/delete! :model/CuratedSearchEntry :entity_id sli-eid)))))))))
+
+(deftest unmapped-entity-model-passes-through-test
+ (testing "an unrecognized entity model exports as-is (raw id, no deps) instead of aborting the export"
+ ;; The CRUD API rejects unknown models, but rows written before a model string was retired (or by
+ ;; direct appdb writes) shouldn't take down a whole export.
+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id}
+ {:search_prompt "legacy" :entity {:model "garbage" :id 5}}]
+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)]
+ (is (=? {:entity {:model "garbage" :id 5}} extracted))
+ (is (= #{} (serdes/dependencies extracted)))))))
+
+(deftest table-entity-round-trip-test
+ (let [table-id (mt/id :venues)]
+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id sli-eid :entity_id}
+ {:search_prompt "best venues" :entity {:model "table" :id table-id}}]
+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)
+ table-ref (:entity extracted)]
+ (testing "the table ref is exported as a [db schema table] path"
+ (is (=? {:entity_id sli-eid
+ :entity {:model "table" :id vector?}}
+ extracted))
+ (is (= ["VENUES"] (take-last 1 (:id table-ref)))))
+ (testing "dependencies cover the Table's Database (the Table is synthesized on import)"
+ (is (= #{[{:model "Database" :id (first (:id table-ref))}]}
+ (serdes/dependencies extracted))))
+ (testing "importing resolves the ref back to the local id"
+ (t2/delete! :model/CuratedSearchEntry :id sli-id)
+ (serdes.load/load-metabase! (ingestion-in-memory [extracted]))
+ (is (= {:model "table" :id table-id}
+ (t2/select-one-fn :entity :model/CuratedSearchEntry :entity_id sli-eid)))
+ ;; load-metabase! re-inserts under a new id that with-temp can't reap; clean it up so the row
+ ;; doesn't leak into other tests' view of the (small, fully-scanned) table.
+ (t2/delete! :model/CuratedSearchEntry :entity_id sli-eid))))))
+
+(deftest disk-export-import-round-trip-test
+ (testing "CuratedSearchEntry survives a real on-disk export/import"
+ ;; The in-memory tests above skip two layers that have silently dropped the model before:
+ ;; `extract/model-set` (the default-export selection) and `ingest/legal-top-level-paths` (which
+ ;; dirs the importer reads). This drives both through actual YAML on disk.
+ (let [serialized (atom nil)
+ sli-eid (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "Prompt DB")
+ t1 (ts/create! :model/Table :name "ORDERS" :db_id (:id db) :schema "PUBLIC")
+ sli (ts/create! :model/CuratedSearchEntry
+ :search_prompt "orders" :verified true
+ :entity {:model "table" :id (:id t1)})]
+ (reset! sli-eid (:entity_id sli))
+ ;; realize inside with-cache so the cached resolvers are actually used during extraction
+ (reset! serialized (serdes/with-cache (into [] (extract/extract {}))))))
+ (testing "default export selection includes it (guards extract/model-set)"
+ (is (= 1 (count (filter (fn [{[{:keys [model]}] :serdes/meta}]
+ (= model "CuratedSearchEntry"))
+ @serialized)))))
+ (ts/with-random-dump-dir [dump-dir "sli-serdes-"]
+ (storage/store! (seq @serialized) (storage.files/file-writer dump-dir))
+ (testing "stored under the curated_search_entries/ directory"
+ (is (.exists (io/file dump-dir "curated_search_entries"))))
+ (testing "ingest reads it back (guards ingest/legal-top-level-paths)"
+ (is (contains? (set (serdes.ingest/ingest-list (serdes.ingest/ingest-yaml dump-dir)))
+ [{:model "CuratedSearchEntry" :id @sli-eid}])))
+ (testing "loads into a fresh appdb with the table ref resolved"
+ (ts/with-db dest-db
+ (serdes/with-cache (serdes.load/load-metabase! (serdes.ingest/ingest-yaml dump-dir)))
+ (let [row (t2/select-one :model/CuratedSearchEntry :entity_id @sli-eid)]
+ (is (=? {:verified true} row))
+ (is (= "ORDERS"
+ (t2/select-one-fn :name :model/Table :id (get-in row [:entity :id]))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index ea6085e1b92..5a496c3a32d 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -131,6 +131,12 @@
(def ^:private internal-model?
#{"Schema"})
+(def ^:private covered-by-dedicated-round-trip-test?
+ "Models that have full export/import coverage in their own round-trip test and so don't need a
+ fixture in this shared baseline. CuratedSearchEntry is covered (in-memory + on-disk) by
+ metabase-enterprise.serialization.v2.curated-search-entry-test."
+ #{"CuratedSearchEntry"})
+
(defn add-to-baseline!
"Use this within v2.extract-test where relevant to add their fixtures to the baseline."
[]
@@ -142,7 +148,7 @@
resources (ingest/ingest-list ingestable)
baselined (into #{} (map :model) (apply concat resources))
necessary? (set serdes.models/exported-models)]
- (doseq [m serdes.models/exported-models]
+ (doseq [m serdes.models/exported-models :when (not (covered-by-dedicated-round-trip-test? m))]
(is (baselined m) (format "We need to add %s entries to %s" m source-dir-path)))
(doseq [b baselined :when (not (internal-model? b))]
(is (necessary? b) (format "We can remove %s files from %s" b source-dir-path)))))
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index e15ce6f330a..6a2da90ca06 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -104,6 +104,7 @@
collections
config
content-verification
+ curated-search
custom-viz-plugin
dashboards
documents
diff --git a/resources/migrations/063/20260604_curated_search_entries.yaml b/resources/migrations/063/20260604_curated_search_entries.yaml
new file mode 100644
index 00000000000..251d0f1e7ac
--- /dev/null
+++ b/resources/migrations/063/20260604_curated_search_entries.yaml
@@ -0,0 +1,72 @@
+## curated_search_entries: curated saved-search-prompt -> entity rows. The table is authoritative;
+## an enterprise pgvector mirror (one embedding per row) backs the `curated_search` Metabot tool.
+
+databaseChangeLog:
+ - logicalFilePath: migrations/063_update_migrations.yaml
+
+ - changeSet:
+ id: v63.2026-06-04T00:00:00
+ author: christruter
+ comment: Create curated_search_entries table
+ changes:
+ - createTable:
+ tableName: curated_search_entries
+ remarks: Curated search prompts mapped to the entities that answer them
+ columns:
+ - column:
+ name: id
+ type: int
+ remarks: Unique ID
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: search_prompt
+ type: ${text.type}
+ remarks: The saved search prompt text
+ constraints:
+ nullable: false
+ - column:
+ name: usage_instructions
+ type: ${text.type}
+ remarks: Agent-facing guidance on how to use the mapped entities
+ constraints:
+ nullable: true
+ - column:
+ name: entity
+ type: ${text.type}
+ remarks: JSON entity ref ({model, id, name?}) the prompt maps to
+ constraints:
+ nullable: false
+ - column:
+ name: verified
+ type: ${boolean.type}
+ remarks: Whether this prompt-entity mapping has been verified by a curator
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ - column:
+ name: entity_id
+ type: char(21)
+ remarks: Random NanoID tag for unique identity (serialization)
+ constraints:
+ nullable: true
+ unique: true
+ - column:
+ name: created_at
+ type: ${timestamp_type}
+ remarks: When the mapping was created
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: updated_at
+ type: ${timestamp_type}
+ remarks: When the mapping was last updated
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ rollback:
+ - dropTable:
+ tableName: curated_search_entries
diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.clj
index b9516e5e6e4..bce0045ed01 100644
--- a/src/metabase/api_routes/routes.clj
+++ b/src/metabase/api_routes/routes.clj
@@ -21,6 +21,7 @@
[metabase.collections-rest.api]
[metabase.comments.api]
[metabase.config.core :as config]
+ [metabase.curated-search.api]
[metabase.dashboards-rest.api]
[metabase.data-studio.api]
[metabase.documents.api]
@@ -88,6 +89,7 @@
metabase.cloud-migration.api/keep-me
metabase.comments.api/keep-me
metabase.collections-rest.api/keep-me
+ metabase.curated-search.api/keep-me
metabase.dashboards-rest.api/keep-me
metabase.data-studio.api/keep-me
metabase.documents.api/keep-me
@@ -178,6 +180,7 @@
"/cloud-migration" (+auth 'metabase.cloud-migration.api)
"/collection" (+auth 'metabase.collections-rest.api)
"/comment" (+auth metabase.comments.api/routes)
+ "/curated-search" (+auth 'metabase.curated-search.api)
"/dashboard" (+auth 'metabase.dashboards-rest.api)
"/data-studio" (+auth metabase.data-studio.api/routes)
"/database" (+auth 'metabase.warehouses-rest.api)
diff --git a/src/metabase/cmd/copy.clj b/src/metabase/cmd/copy.clj
index 175f351ce60..e595a435cb2 100644
--- a/src/metabase/cmd/copy.clj
+++ b/src/metabase/cmd/copy.clj
@@ -138,7 +138,8 @@
:model/MetabotFeedback
:model/MetabotSourceFeedback
:model/MetabotUsedTable
- :model/MetabotPrompt]
+ :model/MetabotPrompt
+ :model/CuratedSearchEntry]
(when config/ee-available?
[:model/MetabotPermissions
:model/MetabotGroupLimit
diff --git a/src/metabase/curated_search/api.clj b/src/metabase/curated_search/api.clj
new file mode 100644
index 00000000000..01e1ff20032
--- /dev/null
+++ b/src/metabase/curated_search/api.clj
@@ -0,0 +1,117 @@
+(ns metabase.curated-search.api
+ "Admin REST API for managing curated search entries: curated search prompts mapped to the entities
+ that answer them."
+ (:require
+ [metabase.api.common :as api]
+ [metabase.api.macros :as api.macros]
+ [metabase.request.core :as request]
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2]))
+
+(def ^:private EntityRef
+ "Entity ref as accepted on writes: the model must be a known agent-facing entity type (as used with
+ read_resource), plus plain \"card\"."
+ [:map
+ [:model [:enum "table" "card" "model" "metric" "question"]]
+ [:id ms/PositiveInt]
+ [:name {:optional true} :string]])
+
+(def ^:private EntityRefOut
+ "Entity ref as returned on reads. `:model` is any string: rows can predate a model string's retirement
+ (serdes tolerates those too), and one legacy row must not fail response validation for a whole list."
+ [:map
+ [:model :string]
+ [:id :int]
+ [:name {:optional true} :string]])
+
+(def ^:private Entry
+ [:map
+ [:id ms/PositiveInt]
+ [:search_prompt :string]
+ [:usage_instructions {:optional true} [:maybe :string]]
+ [:entity EntityRefOut]
+ [:verified :boolean]])
+
+(def ^:private default-limit 50)
+(def ^:private default-offset 0)
+
+(api.macros/defendpoint :get "/"
+ :- [:map
+ [:data [:sequential Entry]]
+ [:total :int]
+ [:limit :int]
+ [:offset :int]]
+ "Get all curated search entries, paginated."
+ [_route-params
+ _query-params]
+ (api/check-superuser)
+ (let [limit (or (request/limit) default-limit)
+ offset (or (request/offset) default-offset)]
+ {:data (t2/select :model/CuratedSearchEntry
+ {:order-by [[:id :asc]]
+ :limit limit
+ :offset offset})
+ :total (t2/count :model/CuratedSearchEntry)
+ :limit limit
+ :offset offset}))
+
+(api.macros/defendpoint :get "/:id"
+ :- Entry
+ "Get a curated search entry by ID."
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ _query-params]
+ (api/check-superuser)
+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id)))
+
+(api.macros/defendpoint :post "/"
+ :- Entry
+ "Create a new curated search entry."
+ [_route-params
+ _query-params
+ {:keys [search_prompt usage_instructions entity verified]}
+ :- [:map
+ [:search_prompt ms/NonBlankString]
+ [:usage_instructions {:optional true} [:maybe :string]]
+ [:entity EntityRef]
+ [:verified {:optional true} [:maybe :boolean]]]]
+ (api/check-superuser)
+ (t2/insert-returning-instance! :model/CuratedSearchEntry
+ {:search_prompt search_prompt
+ :usage_instructions usage_instructions
+ :entity entity
+ :verified (boolean verified)}))
+
+(api.macros/defendpoint :put "/:id"
+ :- Entry
+ "Update a curated search entry by ID. Only the provided fields are changed; sending a null
+ usage_instructions clears it."
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ _query-params
+ ;; patch semantics: presence of a key (not nil-ness of its value) decides what changes, so an explicit
+ ;; null can clear the nullable usage_instructions. The non-nullable fields reject null in the schema.
+ {:keys [search_prompt usage_instructions entity verified] :as body}
+ :- [:map
+ [:search_prompt {:optional true} ms/NonBlankString]
+ [:usage_instructions {:optional true} [:maybe :string]]
+ [:entity {:optional true} EntityRef]
+ [:verified {:optional true} :boolean]]]
+ (api/check-superuser)
+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id))
+ (let [changes (cond-> {}
+ (contains? body :search_prompt) (assoc :search_prompt search_prompt)
+ (contains? body :usage_instructions) (assoc :usage_instructions usage_instructions)
+ (contains? body :entity) (assoc :entity entity)
+ (contains? body :verified) (assoc :verified verified))]
+ (when (seq changes)
+ (t2/update! :model/CuratedSearchEntry id changes)))
+ (t2/select-one :model/CuratedSearchEntry :id id))
+
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :delete "/:id"
+ "Delete a curated search entry by ID."
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]
+ _query-params]
+ (api/check-superuser)
+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id))
+ (t2/delete! :model/CuratedSearchEntry :id id)
+ api/generic-204-no-content)
diff --git a/src/metabase/curated_search/core.clj b/src/metabase/curated_search/core.clj
new file mode 100644
index 00000000000..d02732f02d6
--- /dev/null
+++ b/src/metabase/curated_search/core.clj
@@ -0,0 +1,16 @@
+(ns metabase.curated-search.core
+ "Public API of the `curated-search` module.
+
+ The curated search index is a hand-curated library of saved search prompts, each mapped to the entity that
+ answers it (see [[metabase.curated-search.models.curated-search-entry]]).
+ [[search]] matches a natural-language request against it by vector similarity (enterprise; returns []
+ in OSS)."
+ (:require
+ [metabase.curated-search.mirror]
+ [potemkin :as p]))
+
+(comment metabase.curated-search.mirror/keep-me)
+
+(p/import-vars
+ [metabase.curated-search.mirror
+ search])
diff --git a/src/metabase/curated_search/mirror.clj b/src/metabase/curated_search/mirror.clj
new file mode 100644
index 00000000000..5bca05b6b0d
--- /dev/null
+++ b/src/metabase/curated_search/mirror.clj
@@ -0,0 +1,24 @@
+(ns metabase.curated-search.mirror
+ "OSS entry points for the enterprise pgvector mirror of the `curated_search_entries` table.
+
+ The appdb table is authoritative; the mirror carries one embedding per row and serves the
+ `search_curated` Metabot tool's similarity search.
+ These `defenterprise` shims route to [[metabase-enterprise.curated-search.core]] when an
+ enterprise license with semantic search is present; otherwise they no-op or return empty."
+ (:require
+ [metabase.premium-features.core :refer [defenterprise]]))
+
+(defenterprise request-sync!
+ "Ask the background sync to reconcile the pgvector mirror with the appdb table soon; no-op in OSS.
+ Fire-and-forget: never throws, does no embedding or pgvector work on the calling thread."
+ metabase-enterprise.curated-search.core
+ []
+ nil)
+
+(defenterprise search
+ "Similarity-search the curated search index for the saved prompts nearest `user-search-prompt`.
+ Returns up to `limit` matches shaped `{:saved_search_prompt :usage_instructions :entity :score}`,
+ best blended score first; [] in OSS or when no pgvector store is configured."
+ metabase-enterprise.curated-search.core
+ [_user-search-prompt _limit]
+ [])
diff --git a/src/metabase/curated_search/models/curated_search_entry.clj b/src/metabase/curated_search/models/curated_search_entry.clj
new file mode 100644
index 00000000000..d043176df17
--- /dev/null
+++ b/src/metabase/curated_search/models/curated_search_entry.clj
@@ -0,0 +1,102 @@
+(ns metabase.curated-search.models.curated-search-entry
+ "The `curated_search_entries` appdb table: curated saved search prompts, each mapped to the single entity
+ that answers it, plus agent-facing `usage_instructions`.
+
+ This table is authoritative.
+ An enterprise pgvector mirror carries one embedding per row and serves the `search_curated`
+ Metabot tool's similarity search.
+ Writes here only nudge the mirror's background sync ([[mirror/request-sync!]]); they never touch
+ the embedding service or the pgvector store themselves."
+ (:require
+ [metabase.curated-search.mirror :as mirror]
+ [metabase.models.interface :as mi]
+ [metabase.models.serialization :as serdes]
+ [metabase.util :as u]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/CuratedSearchEntry [_model] :curated_search_entries)
+
+(doto :model/CuratedSearchEntry
+ (derive :metabase/model)
+ (derive :hook/entity-id)
+ (derive :hook/timestamped?))
+
+(t2/deftransforms :model/CuratedSearchEntry
+ {:entity mi/transform-json})
+
+;;; Each write nudges the enterprise background sync, which reconciles the pgvector mirror against this
+;;; table. The nudge is fire-and-forget (no-op in OSS, error-swallowing in EE), so appdb writes never
+;;; fail or slow down because of the mirror.
+
+(t2/define-after-insert :model/CuratedSearchEntry
+ [row]
+ (mirror/request-sync!)
+ row)
+
+(t2/define-after-update :model/CuratedSearchEntry
+ [row]
+ (mirror/request-sync!)
+ row)
+
+(t2/define-before-delete :model/CuratedSearchEntry
+ [row]
+ (mirror/request-sync!)
+ row)
+
+;;; ------------------------------------------------- Serialization -------------------------------------------------
+
+;;; `entity` is a polymorphic reference — `{:model "table"|"card"|"model"|"metric"|"question" :id <local-id>}`.
+;;; On export we swap the local id for a portable reference (a Card's entity_id, or a Table's
+;;; `[db schema table]` path) and reverse it on import, mirroring `serdes/export-viz-link-card`.
+
+(def ^:private entity-model->toucan
+ "Toucan model for each entity ref `:model` string. Tables get the table-fk treatment instead.
+ All the card flavors are Cards (the `:model` string mirrors the `read_resource` resource type the
+ agent uses to fetch the entity, not the underlying table)."
+ {"card" :model/Card
+ "model" :model/Card
+ "metric" :model/Card
+ "question" :model/Card})
+
+(defn- export-entity-ref [{:keys [model id] :as entity}]
+ (cond
+ (= model "table") (assoc entity :id (serdes/*export-table-fk* id))
+ (entity-model->toucan model) (assoc entity :id (serdes/*export-fk* id (entity-model->toucan model)))
+ ;; Unmapped model string: leave the ref as-is rather than aborting the whole export
+ ;; (same forgiving behavior as serdes/export-viz-link-card).
+ :else entity))
+
+(defn- import-entity-ref [{:keys [model id] :as entity}]
+ (cond
+ (= model "table") (assoc entity :id (serdes/*import-table-fk* id))
+ (entity-model->toucan model) (assoc entity :id (serdes/*import-fk* id (entity-model->toucan model)))
+ :else entity))
+
+(defmethod serdes/make-spec "CuratedSearchEntry" [_model-name _opts]
+ {:copy [:entity_id :search_prompt :usage_instructions :verified]
+ :transform {:created_at (serdes/date)
+ :updated_at (serdes/date)
+ :entity {:export export-entity-ref
+ :import import-entity-ref}}
+ :defaults {:verified false}})
+
+(defmethod serdes/hash-fields :model/CuratedSearchEntry
+ [_model]
+ [:search_prompt])
+
+(defmethod serdes/generate-path "CuratedSearchEntry" [_ entity]
+ (serdes/maybe-labeled "CuratedSearchEntry" entity :search_prompt))
+
+(defmethod serdes/storage-path "CuratedSearchEntry" [entity _ctx]
+ [{:label "curated_search_entries"}
+ {:label (u/slugify (:search_prompt entity) {:unicode? true}) :key (:entity_id entity)}])
+
+(defmethod serdes/dependencies "CuratedSearchEntry"
+ [{{:keys [model id]} :entity}]
+ ;; A referenced Table is synthesized on import if missing, so (like link cards) we depend on its Database,
+ ;; not the Table itself. Card refs depend on the Card directly.
+ (cond
+ (= model "table") #{[{:model "Database" :id (first id)}]}
+ (entity-model->toucan model) #{[{:model "Card" :id id}]}
+ :else #{}))
diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.clj
index 486c960317d..e6c9b91b7bd 100644
--- a/src/metabase/metabot/agent/profiles.clj
+++ b/src/metabase/metabot/agent/profiles.clj
@@ -170,7 +170,10 @@
:prompt-template "natural-language-querying-only.selmer"
:max-iterations 10
:temperature 0.3
+ ;; the nlq profile gets both the general instance search and, for users with the semantic-search
+ ;; feature, the curated search tool (gated via its :feature-semantic-search capability).
:tools [#'tools/nlq-search-tool
+ #'tools/curated-search-tool
#'tools/read-resource-tool
#'tools/construct-notebook-query-tool
#'tools/navigate-user-tool
diff --git a/src/metabase/metabot/capabilities.clj b/src/metabase/metabot/capabilities.clj
index 95a3f75b7da..5e9477ab84c 100644
--- a/src/metabase/metabot/capabilities.clj
+++ b/src/metabase/metabot/capabilities.clj
@@ -39,7 +39,9 @@
(premium-features/has-feature? :transforms-basic))
:feature-transforms)
(when (premium-features/has-feature? :transforms-python)
- :feature-transforms-python)]))
+ :feature-transforms-python)
+ (when (premium-features/has-feature? :semantic-search)
+ :feature-semantic-search)]))
(defn capability-set
"Build the full set of capability keywords for a request: the backend-inferred feature capabilities
diff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.clj
index f4707e9b0c2..f620aefd450 100644
--- a/src/metabase/metabot/tools.clj
+++ b/src/metabase/metabot/tools.clj
@@ -15,6 +15,7 @@
[metabase.metabot.tools.construct :as tools.construct]
[metabase.metabot.tools.create-alert :as tools.create-alert]
[metabase.metabot.tools.create-dashboard-subscription :as tools.create-dashboard-subscription]
+ [metabase.metabot.tools.curated-search :as tools.curated-search]
[metabase.metabot.tools.document :as tools.document]
[metabase.metabot.tools.metadata :as tools.metadata]
[metabase.metabot.tools.navigation :as tools.navigation]
@@ -41,6 +42,8 @@
sql-search-tool
nlq-search-tool
transform-search-tool]
+ [tools.curated-search
+ curated-search-tool]
[tools.construct
construct-notebook-query-tool]
[tools.document
diff --git a/src/metabase/metabot/tools/curated_search.clj b/src/metabase/metabot/tools/curated_search.clj
new file mode 100644
index 00000000000..9520b8b348e
--- /dev/null
+++ b/src/metabase/metabot/tools/curated_search.clj
@@ -0,0 +1,180 @@
+(ns metabase.metabot.tools.curated-search
+ "Metabot `search_curated` tool, backed by the curated search index.
+
+ Instead of ranking the whole instance, it matches the user's request by vector similarity against a
+ hand-curated library of saved search prompts, each mapped to the single entity that answers it.
+ Each match also carries `usage_instructions`: curator guidance on how to use that entity.
+
+ The index has several prompts per entity, so raw vector hits are deduped to distinct entities and a
+ small number returned.
+ The matched entity refs are hydrated into the same enriched search-result shape the general `search`
+ tool returns (`portable_entity_id`, fully-qualified names, database names, metric base tables), so the
+ agent can build a query inline without an extra `read_resource` round-trip.
+
+ Even where a general search is also offered, the tool surfaces the raw cosine `similarity` of each
+ match and flags low-confidence results, so a curated miss reads as a miss rather than a
+ confident-but-wrong curated hit.
+ The similarity search runs in the enterprise pgvector store via [[metabase.curated-search.core]]."
+ (:require
+ [clojure.string :as str]
+ [metabase.curated-search.core :as curated-search]
+ [metabase.metabot.scope :as scope]
+ [metabase.metabot.tools.search :as tools.search]
+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private default-limit
+ "Distinct curated matches returned by default. The index has ~3 prompts per entity and embeddings cluster
+ tightly by domain; 5 was too aggressive (benchmark 15: ~12 expected entities sat at rank 6–10 and were
+ cut off), so 10 to recover that recall while still deduping siblings."
+ 10)
+(def ^:private max-limit 20)
+
+(def ^:private over-fetch-factor
+ "Raw per-prompt hits to pull before deduping to distinct entities (several prompts map to one entity)."
+ 4)
+(def ^:private over-fetch-cap 60)
+
+(def ^:private weak-similarity-threshold
+ "Raw cosine similarity below which a match is flagged low-confidence. Calibrated on this index:
+ in-domain top-1 similarities run ~0.52–0.81, clearly out-of-domain ~0.15–0.22."
+ 0.45)
+
+(def ^:private user-search-prompt-desc
+ (str "A natural-language description of the data the user wants. Matched by vector similarity "
+ "against curated saved prompts; phrase it like the saved prompt you'd expect to find, "
+ "not as keywords."))
+
+(def ^:private limit-desc
+ (str "Maximum number of distinct curated matches (default " default-limit ", max " max-limit ")."))
+
+(def ^:private curated-search-schema
+ [:map {:closed true}
+ [:user_search_prompt [:string {:description user-search-prompt-desc}]]
+ [:limit {:optional true} [:maybe [:int {:min 1 :max max-limit :description limit-desc}]]]])
+
+(def ^:private instructions
+ ;; refers to the match/note elements by name only — literal angle-bracket tags here would make the
+ ;; surrounding <instructions> element malformed
+ (str "Each match element is a curated entry: a saved prompt, its usage_instructions, and the entity to "
+ "use (already hydrated with the ids, names and portable references you need). similarity is the "
+ "raw cosine match strength; a match flagged confidence=\"weak\" (or a leading note element) means "
+ "nothing in the curated layer clearly matches — don't build on it blindly; prefer asking the user "
+ "to clarify or narrow the request, or use the general search tool for uncurated results."))
+
+(defn- similarity
+ "Raw cosine similarity (1 - distance) for a match, from its score breakdown."
+ ^double [score]
+ (or (some (fn [s] (when (= :similarity (:name s)) (double (:score s)))) (:scores score)) 0.0))
+
+(defn- dedupe-by-entity
+ "Collapse raw per-prompt results (sorted best-first) to distinct entities, keeping the best-scoring
+ prompt for each."
+ [results]
+ (->> results
+ (reduce (fn [[seen acc] r]
+ ;; normalize the model so "card" and "question" (aliases for the same Card) dedupe
+ ;; together — both hydrate to one record, so distinct keys would return it twice.
+ (let [{:keys [model id]} (:entity r)
+ k [(tools.search/ref-model->entity-type model) id]]
+ (if (seen k) [seen acc] [(conj seen k) (conj acc r)])))
+ [#{} []])
+ second))
+
+(defn- build-matches
+ "Fetch, dedupe to distinct entities, hydrate each match's entity ref into a full search record, and
+ return the top `n` the current user can read.
+ Each match: `{:saved_search_prompt :usage_instructions :score :similarity :weak? :entity hydrated-hit}`."
+ [user-search-prompt n]
+ (let [raw (curated-search/search
+ user-search-prompt (min over-fetch-cap (* over-fetch-factor n)))
+ deduped (dedupe-by-entity raw)
+ ;; Hydration permission-filters (entity-refs->search-results drops entities the user can't read),
+ ;; so hydrate the whole deduped candidate set and take `n` from the readable survivors — taking
+ ;; `n` first would let unreadable top hits shrink the result below `n` while readable matches sit
+ ;; just past the cut.
+ by-key (into {} (map (juxt (juxt :type :id) identity))
+ (tools.search/entity-refs->search-results (distinct (map :entity deduped))))]
+ (->> (for [{:keys [saved_search_prompt usage_instructions entity score]} deduped
+ ;; hydrated records use the agent-facing entity type, so normalize the ref's model to match
+ ;; (plain "card" refs hydrate as "question")
+ :let [resolved (get by-key [(tools.search/ref-model->entity-type (:model entity)) (:id entity)])
+ sim (similarity score)]
+ :when resolved]
+ {:saved_search_prompt saved_search_prompt
+ :usage_instructions usage_instructions
+ :score score
+ :similarity sim
+ :weak? (< sim weak-similarity-threshold)
+ :entity resolved})
+ (take n))))
+
+(defn- match->xml [{:keys [saved_search_prompt usage_instructions score similarity weak? entity]}]
+ ;; String/format with Locale/ROOT so %.3f uses a '.' decimal separator regardless of the instance's
+ ;; site locale (clojure.core/format always uses the default locale).
+ (str (String/format java.util.Locale/ROOT "<match score=\"%.3f\" similarity=\"%.3f\" confidence=\"%s\">\n"
+ (object-array [(double (:total_score score)) (double similarity)
+ (if weak? "weak" "strong")]))
+ "<saved_search_prompt>" (llm-shape/escape-xml saved_search_prompt) "</saved_search_prompt>\n"
+ (when-not (str/blank? usage_instructions)
+ (str "<usage_instructions>" (llm-shape/escape-xml usage_instructions) "</usage_instructions>\n"))
+ (llm-shape/search-result->xml entity)
+ "\n</match>"))
+
+(defn- format-output [matches]
+ (if (empty? matches)
+ "<search_results>No matching curated entities.</search_results>"
+ (str (when (:weak? (first matches))
+ (str "<note>No strong curated match for this request — treat the results below as weak "
+ "guesses; consider asking the user to clarify or narrow the request.</note>\n"))
+ "<search_results>\n"
+ (str/join "\n" (map match->xml matches))
+ "\n</search_results>\n"
+ "<instructions>" instructions "</instructions>")))
+
+(defn- flatten-data
+ "Flatten matches to a per-entity list (the `:result-type :search` shape), each entity record annotated
+ with its match's prompt, usage_instructions, similarity and confidence."
+ [matches]
+ (mapv (fn [{:keys [saved_search_prompt usage_instructions score similarity weak? entity]}]
+ (assoc entity
+ :saved_search_prompt saved_search_prompt
+ :usage_instructions usage_instructions
+ :score score
+ :similarity similarity
+ :confidence (if weak? "weak" "strong")))
+ matches))
+
+(mu/defn ^{:tool-name "search_curated"
+ :scope scope/agent-search
+ ;; only offered to users whose license includes semantic search (the mirror's backing store);
+ ;; :feature-semantic-search is derived in profiles/feature-capabilities.
+ :capabilities #{:feature-semantic-search}}
+ curated-search-tool
+ "Find the best data to answer the user's request from the curated search index — a vetted library of
+ saved prompts, each mapped to the entity that answers it. Phrase `user_search_prompt` as a full
+ natural-language description of the data wanted (it is matched on meaning, not keywords).
+
+ Prefer this over the general `search` tool: try it first, and fall back to `search` only when no
+ curated match is strong, or to complement a curated match with broader (uncurated) results.
+
+ Returns a handful of distinct curated matches, best-first. Each match has `saved_search_prompt`, curator
+ `usage_instructions`, a raw `similarity`, and the `entity` — a full search record (name, type, database,
+ `portable_entity_id`, fully-qualified name) you can use directly. If the top match is flagged
+ low-confidence (a leading <note> / confidence=\"weak\"), nothing in the curated layer clearly matches —
+ prefer asking the user to clarify, or fall back to the general `search` tool for uncurated results."
+ [{:keys [user_search_prompt limit]} :- curated-search-schema]
+ (try
+ (let [n (min max-limit (or limit default-limit))
+ matches (build-matches user_search_prompt n)]
+ {:output (format-output matches)
+ :structured-output {:result-type :search
+ :data (flatten-data matches)
+ :total_count (count matches)
+ :weak_match (boolean (:weak? (first matches)))}})
+ (catch Exception e
+ (log/error e "Error in curated search")
+ {:output (str "search_curated failed: " (or (ex-message e) "Unknown error"))})))
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 716a6d28716..51feec73c2d 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -11,6 +11,7 @@
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
[metabase.search.core :as search]
[metabase.search.engine :as search.engine]
@@ -307,6 +308,81 @@
enrich-with-metric-base-tables
remove-unreadable-transforms)))
+(defn- table-refs->results
+ [ids]
+ (when (seq ids)
+ ;; only surface tables the current user can read — a curated entry may point at one they can't access
+ (for [t (filter mi/can-read?
+ (t2/select [:model/Table :id :name :display_name :db_id :schema :description] :id [:in ids]))]
+ {:id (:id t)
+ :type "table"
+ :name (:name t)
+ :display_name (:display_name t)
+ :database_id (:db_id t)
+ :database_schema (:schema t)
+ :description (:description t)})))
+
+(defn- card-refs->results
+ "Build post-processed search-result records for card-backed refs (`{:id .. :type \"model\"|\"metric\"|\"question\"}`).
+ Emits one record per ref, so the same card registered under two type strings yields a record for each
+ (rather than collapsing to one and silently dropping the other)."
+ [refs]
+ (let [ids (distinct (map :id refs))
+ ;; only surface cards the current user can read (collection perms) — see table-refs->results
+ id->card (when (seq ids)
+ (into {} (map (juxt :id identity))
+ (filter mi/can-read?
+ (t2/select [:model/Card :id :name :description :database_id :collection_id :card_schema]
+ :id [:in ids]))))
+ coll-ids (->> (vals id->card) (keep :collection_id) distinct)
+ id->coll (when (seq coll-ids)
+ (into {} (map (juxt :id identity))
+ (t2/select [:model/Collection :id :name :authority_level] :id [:in coll-ids])))
+ ;; verified is already a set (t2/select-fn-set), possibly nil when there were no ids
+ verified (when (seq ids)
+ (t2/select-fn-set :moderated_item_id :model/ModerationReview
+ :moderated_item_id [:in ids] :moderated_item_type "card"
+ :most_recent true :status "verified"))]
+ (for [{:keys [id type]} refs
+ :let [c (id->card id)]
+ :when c]
+ (let [coll (get id->coll (:collection_id c))]
+ {:id id
+ :type type
+ :name (:name c)
+ :description (:description c)
+ :database_id (:database_id c)
+ :verified (contains? verified id)
+ :collection (when coll (select-keys coll [:id :name :authority_level]))}))))
+
+(defn ref-model->entity-type
+ "Normalize an entity ref's `:model` string to the agent-facing entity type: plain cards are
+ `\"question\"` everywhere the agent sees them (`read_resource` URIs, search results)."
+ [model]
+ (if (= model "card") "question" model))
+
+(defn entity-refs->search-results
+ "Hydrate semantic-layer entity refs into the enriched search-result shape that
+ [[metabase.metabot.tools.shared.llm-shape/search-results->xml]] and the `search` tool consume.
+
+ `refs` is a seq of `{:model <entity-type> :id <id>}` where `<entity-type>` is `\"table\"`, `\"model\"`,
+ `\"metric\"`, or `\"question\"` (the names the agent uses with `read_resource`); `\"card\"` is accepted
+ and normalized to `\"question\"`.
+ Returns records carrying `:portable_entity_id`, `:database_name`, fully-qualified names, metric base
+ tables, etc. — everything the LLM needs to build a query without an extra round-trip.
+ Refs whose entity no longer exists are dropped."
+ [refs]
+ (let [by-model (group-by (comp ref-model->entity-type :model) refs)
+ table-ids (distinct (map :id (get by-model "table")))
+ card-refs (for [m ["model" "metric" "question"], r (get by-model m)] {:id (:id r) :type m})]
+ (->> (concat (table-refs->results table-ids)
+ (card-refs->results (distinct card-refs)))
+ enrich-with-collection-descriptions
+ enrich-with-database-engines
+ enrich-with-portable-entity-ids
+ enrich-with-metric-base-tables
+ remove-unreadable-transforms)))
+
(defn- format-search-output
"Format search results as an LLM-ready string."
[results]
diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.clj
index 35733ef1a12..dce8ed5bd80 100644
--- a/src/metabase/metabot/tools/shared/llm_shape.clj
+++ b/src/metabase/metabot/tools/shared/llm_shape.clj
@@ -73,9 +73,10 @@
(seq data))
(str "```json\n" (json/encode data {:pretty true}) "\n```")))
-(defn- escape-xml
+(defn escape-xml
"Escape XML special characters in a string.
- Only needed for content that bypasses Selmer's auto-escaping (marked with |safe)."
+ Only needed for content that bypasses Selmer's auto-escaping (marked with |safe) or is interpolated
+ into hand-built XML."
[s]
(when s
(-> (str s)
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 0ef42e1e353..35498a35801 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -33,6 +33,7 @@
:model/CommentReaction metabase.comments.models.comment-reaction
:model/ConnectionImpersonation metabase-enterprise.impersonation.models
:model/ContentTranslation metabase.content-translation.models
+ :model/CuratedSearchEntry metabase.curated-search.models.curated-search-entry
:model/CustomVizPlugin metabase-enterprise.custom-viz-plugin.models.custom-viz-plugin
:model/Dashboard metabase.dashboards.models.dashboard
:model/DashboardBookmark metabase.bookmarks.models.bookmark
diff --git a/test/metabase/curated_search/api_test.clj b/test/metabase/curated_search/api_test.clj
new file mode 100644
index 00000000000..2b4ad211fcf
--- /dev/null
+++ b/test/metabase/curated_search/api_test.clj
@@ -0,0 +1,129 @@
+(ns metabase.curated-search.api-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defmacro ^:private with-test-entry
+ "Create a curated search entry for testing and bind it to `sym`, deleting it on exit."
+ [[sym attrs] & body]
+ `(let [~sym (t2/insert-returning-instance! :model/CuratedSearchEntry
+ (merge {:search_prompt "find orders"
+ :entity {:model "table" :id 1}
+ :verified false}
+ ~attrs))]
+ (try
+ ~@body
+ (finally
+ (t2/delete! :model/CuratedSearchEntry :id (:id ~sym))))))
+
+(deftest list-test
+ (with-test-entry [entry {}]
+ (testing "superuser can list curated search entries"
+ (let [response (mt/user-http-request :crowberto :get 200 "curated-search/")]
+ (is (contains? response :data))
+ (is (contains? response :total))
+ (is (some #(= (:id entry) (:id %)) (:data response)))))
+ (testing "non-superuser gets 403"
+ (mt/user-http-request :rasta :get 403 "curated-search/"))
+ (testing "a legacy row whose entity model is no longer in the write enum still reads back fine"
+ (with-test-entry [legacy {:entity {:model "retired-model-string" :id 7}}]
+ (is (some #(= (:id legacy) (:id %))
+ (:data (mt/user-http-request :crowberto :get 200 "curated-search/"))))))))
+
+(deftest get-test
+ (with-test-entry [entry {:search_prompt "find customers" :verified true}]
+ (testing "superuser can fetch a curated search entry by id"
+ (is (=? {:id (:id entry)
+ :search_prompt "find customers"
+ :entity {:model "table" :id 1}
+ :verified true}
+ (mt/user-http-request :crowberto :get 200 (str "curated-search/" (:id entry))))))
+ (testing "returns 404 for unknown id"
+ (mt/user-http-request :crowberto :get 404 "curated-search/0"))
+ (testing "non-superuser gets 403"
+ (mt/user-http-request :rasta :get 403 (str "curated-search/" (:id entry))))))
+
+(deftest create-test
+ (testing "superuser can create a curated search entry"
+ (let [response (mt/user-http-request :crowberto :post 200 "curated-search/"
+ {:search_prompt "find revenue"
+ :usage_instructions "Use the Revenue card."
+ :entity {:model "card" :id 42}})]
+ (try
+ (is (=? {:search_prompt "find revenue"
+ :usage_instructions "Use the Revenue card."
+ :entity {:model "card" :id 42}
+ :verified false}
+ response))
+ (finally
+ (t2/delete! :model/CuratedSearchEntry :id (:id response))))))
+ (testing "verified defaults to false and usage_instructions to nil when omitted"
+ (let [response (mt/user-http-request :crowberto :post 200 "curated-search/"
+ {:search_prompt "find users"
+ :entity {:model "table" :id 1}})]
+ (try
+ (is (false? (:verified response)))
+ (is (nil? (:usage_instructions response)))
+ (finally
+ (t2/delete! :model/CuratedSearchEntry :id (:id response))))))
+ (testing "entity is required"
+ (mt/user-http-request :crowberto :post 400 "curated-search/"
+ {:search_prompt "find users"}))
+ (testing "an unknown entity model is rejected"
+ (mt/user-http-request :crowberto :post 400 "curated-search/"
+ {:search_prompt "find users"
+ :entity {:model "garbage" :id 1}}))
+ (testing "a blank search_prompt is rejected"
+ (mt/user-http-request :crowberto :post 400 "curated-search/"
+ {:search_prompt " " :entity {:model "table" :id 1}}))
+ (testing "non-superuser gets 403"
+ (mt/user-http-request :rasta :post 403 "curated-search/"
+ {:search_prompt "find orders" :entity {:model "table" :id 1}})))
+
+(deftest update-test
+ (with-test-entry [entry {}]
+ (testing "superuser can update the prompt"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (str "curated-search/" (:id entry))
+ {:search_prompt "updated prompt"})]
+ (is (= "updated prompt" (:search_prompt updated)))))
+ (testing "superuser can update usage_instructions and the entity"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (str "curated-search/" (:id entry))
+ {:usage_instructions "Join orders to customers on customer_id."
+ :entity {:model "model" :id 7}})]
+ (is (=? {:usage_instructions "Join orders to customers on customer_id."
+ :entity {:model "model" :id 7}}
+ updated))))
+ (testing "an explicit null clears usage_instructions"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (str "curated-search/" (:id entry))
+ {:usage_instructions nil})]
+ (is (nil? (:usage_instructions updated)))))
+ (testing "the non-nullable fields reject an explicit null"
+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))
+ {:entity nil})
+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))
+ {:verified nil}))
+ (testing "a blank search_prompt is rejected"
+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))
+ {:search_prompt " "}))
+ (testing "returns 404 for unknown id"
+ (mt/user-http-request :crowberto :put 404 "curated-search/0" {:search_prompt "x"}))
+ (testing "non-superuser gets 403"
+ (mt/user-http-request :rasta :put 403
+ (str "curated-search/" (:id entry))
+ {:search_prompt "x"}))))
+
+(deftest delete-test
+ (with-test-entry [entry {}]
+ (testing "non-superuser gets 403"
+ (mt/user-http-request :rasta :delete 403 (str "curated-search/" (:id entry))))
+ (testing "superuser can delete an entry"
+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" (:id entry)))
+ (is (nil? (t2/select-one :model/CuratedSearchEntry :id (:id entry)))))
+ (testing "returns 404 for unknown id"
+ (mt/user-http-request :crowberto :delete 404 "curated-search/0"))))
diff --git a/test/metabase/curated_search/models/curated_search_entry_test.clj b/test/metabase/curated_search/models/curated_search_entry_test.clj
new file mode 100644
index 00000000000..671bc71e23f
--- /dev/null
+++ b/test/metabase/curated_search/models/curated_search_entry_test.clj
@@ -0,0 +1,58 @@
+(ns metabase.curated-search.models.curated-search-entry-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.curated-search.mirror :as mirror]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private table-entity {:model "table" :id 42})
+
+(deftest entity-json-roundtrips-test
+ (testing "entity is a JSON object and timestamps are populated"
+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}
+ {:search_prompt "monthly revenue by region" :entity table-entity}]
+ (is (=? {:search_prompt "monthly revenue by region"
+ :entity table-entity
+ :verified false
+ :created_at some?
+ :updated_at some?}
+ (t2/select-one :model/CuratedSearchEntry :id id))))))
+
+(deftest usage-instructions-persist-test
+ (testing "usage_instructions persists alongside the prompt"
+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}
+ {:search_prompt "monthly revenue by region" :entity table-entity
+ :usage_instructions "Use this metric for total revenue; group by month."}]
+ (is (= "Use this metric for total revenue; group by month."
+ (t2/select-one-fn :usage_instructions :model/CuratedSearchEntry :id id)))))
+ (testing "usage_instructions is optional (nil when omitted)"
+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}
+ {:search_prompt "orders" :entity table-entity}]
+ (is (nil? (t2/select-one-fn :usage_instructions :model/CuratedSearchEntry :id id))))))
+
+(deftest hooks-nudge-the-background-sync-test
+ (testing "insert, update and delete each request a background sync — and do nothing else"
+ (let [nudges (atom 0)]
+ (mt/with-dynamic-fn-redefs [mirror/request-sync! (fn [] (swap! nudges inc) nil)]
+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}
+ {:search_prompt "p1" :entity table-entity}]
+ (is (= 1 @nudges) "insert nudges once")
+ (t2/update! :model/CuratedSearchEntry id {:search_prompt "p1 updated"})
+ (is (= 2 @nudges) "update nudges once")
+ (t2/delete! :model/CuratedSearchEntry :id id)
+ (is (= 3 @nudges) "delete nudges once"))))))
+
+(deftest hooks-never-break-appdb-writes-test
+ (testing "insert/update/delete succeed even though the mirror is unavailable in tests"
+ ;; The hooks call the OSS defenterprise shim, which no-ops without an enterprise license. They must
+ ;; never throw or fail the authoritative write.
+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}
+ {:search_prompt "p1" :entity table-entity}]
+ (testing "update"
+ (is (pos? (t2/update! :model/CuratedSearchEntry id {:search_prompt "p1 updated"})))
+ (is (= "p1 updated" (t2/select-one-fn :search_prompt :model/CuratedSearchEntry :id id))))
+ (testing "delete"
+ (is (pos? (t2/delete! :model/CuratedSearchEntry :id id)))
+ (is (nil? (t2/select-one :model/CuratedSearchEntry :id id)))))))
diff --git a/test/metabase/metabot/agent/profiles_test.clj b/test/metabase/metabot/agent/profiles_test.clj
index 9bc90cff472..af6fd477f3a 100644
--- a/test/metabase/metabot/agent/profiles_test.clj
+++ b/test/metabase/metabot/agent/profiles_test.clj
@@ -64,7 +64,9 @@
(is (= "anthropic/claude-sonnet-4-6" (:model profile)))
(is (= 10 (:max-iterations profile)))
(is (= 0.3 (:temperature profile)))
+ ;; nlq offers both the general search and the curated search tool (the latter feature-gated)
(is (contains? (tool-names profile) "search"))
+ (is (contains? (tool-names profile) "search_curated"))
(is (contains? (tool-names profile) "construct_notebook_query"))))
(testing "retrieves slackbot profile"
(let [profile (profiles/get-profile :slackbot)]
@@ -156,11 +158,12 @@
"SQL tools should be gated by permission:write_sql_queries"))))))
(deftest embedding-next-matches-nlq-tools-test
- (testing "embedding_next and nlq profiles have identical tool sets"
+ (testing "nlq has the same tools as embedding_next plus the (feature-gated) curated search tool"
(let [tool-names (fn [profile] (set (map #(:tool-name (meta %)) (:tools profile))))
embedding (profiles/get-profile :embedding_next)
nlq (profiles/get-profile :nlq)]
- (is (= (tool-names nlq) (tool-names embedding)))))
+ (is (= (disj (tool-names nlq) "search_curated")
+ (tool-names embedding)))))
(binding [scope/*current-user-scope* api-scope/unrestricted]
(testing "navigate_user is excluded without the capability"
(let [tools (profiles/get-tools-for-profile :embedding_next [])]
diff --git a/test/metabase/metabot/tools/curated_search_test.clj b/test/metabase/metabot/tools/curated_search_test.clj
new file mode 100644
index 00000000000..46837448c69
--- /dev/null
+++ b/test/metabase/metabot/tools/curated_search_test.clj
@@ -0,0 +1,146 @@
+(ns metabase.metabot.tools.curated-search-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.curated-search.core :as cs.core]
+ [metabase.metabot.tools.curated-search :as curated-search]
+ [metabase.metabot.tools.search :as tools.search]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(deftest tool-shape-oss-fallback-test
+ (testing "without the semantic-search feature the tool returns the standard empty result shape"
+ ;; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient
+ ;; premium token (e.g. an all-features dev token) or rows left in a local pgvector store.
+ (mt/with-premium-features #{}
+ (is (=? {:output #(str/includes? % "No matching curated entities")
+ :structured-output {:result-type :search
+ :data []
+ :total_count 0
+ :weak_match false}}
+ (curated-search/curated-search-tool
+ {:user_search_prompt "revenue per region"}))))))
+
+(deftest similarity-helper-test
+ (let [similarity (var-get #'curated-search/similarity)]
+ (testing "similarity is pulled from the :similarity score factor"
+ (is (= 0.8 (similarity {:scores [{:name :similarity :score 0.8} {:name :verified :score 1.0}]})))
+ (is (= 0.0 (similarity {:scores [{:name :verified :score 1.0}]}))))))
+
+(deftest build-matches-permission-filters-before-take-test
+ (testing "hydration permission-filters the whole candidate set before take-n, so an unreadable top hit doesn't crowd out readable matches just past the cut"
+ (let [build-matches (var-get #'curated-search/build-matches)
+ ;; three distinct entities, best-first
+ raw (vec (for [id [1 2 3]]
+ {:entity {:model "table" :id id}
+ :saved_search_prompt (str "p" id)
+ :usage_instructions ""
+ :score {:total_score (- 1.0 (/ id 10.0))
+ :scores [{:name :similarity :score 0.8}]}}))]
+ (mt/with-dynamic-fn-redefs [cs.core/search (fn [_ _] raw)
+ ;; entity 1 (the top hit) is unreadable → dropped during hydration
+ tools.search/entity-refs->search-results
+ (fn [refs] (for [{:keys [model id]} refs :when (not= 1 id)]
+ {:type model :id id :name (str "e" id)}))]
+ (testing "asking for 2 returns the 2 readable entities (2 and 3), not just the 1 below the old top-2 cut"
+ (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2)))))))))
+
+(deftest match->xml-locale-independent-test
+ (testing "score/similarity render with a '.' decimal separator even under a comma-decimal default locale"
+ (let [match->xml (var-get #'curated-search/match->xml)
+ orig (java.util.Locale/getDefault)]
+ (try
+ (java.util.Locale/setDefault (java.util.Locale/forLanguageTag "de-DE"))
+ (let [out (match->xml {:saved_search_prompt "p" :usage_instructions ""
+ :score {:total_score 0.95} :similarity 0.78 :weak? false
+ :entity {:type "table" :id 1 :name "T" :database_id 1}})]
+ (is (str/includes? out "score=\"0.950\""))
+ (is (str/includes? out "similarity=\"0.780\"")))
+ (finally
+ (java.util.Locale/setDefault orig))))))
+
+(deftest dedupe-by-entity-test
+ (let [dedupe (var-get #'curated-search/dedupe-by-entity)
+ results [{:entity {:model "metric" :id 9} :score {:total_score 0.95}} ; entity 9, best
+ {:entity {:model "metric" :id 9} :score {:total_score 0.80}} ; entity 9 again (sibling prompt)
+ {:entity {:model "table" :id 1} :score {:total_score 0.70}}]]
+ (testing "collapses to distinct entities, keeping the first (best-scoring) occurrence"
+ (let [out (dedupe results)]
+ (is (= 2 (count out)))
+ (is (= [0.95 0.70] (mapv (comp :total_score :score) out)))))
+ (testing "\"card\" and \"question\" are aliases for the same Card, so they collapse to one entity"
+ (let [out (dedupe [{:entity {:model "card" :id 5} :score {:total_score 0.9}}
+ {:entity {:model "question" :id 5} :score {:total_score 0.8}}])]
+ (is (= 1 (count out)))
+ (is (= 0.9 (-> out first :score :total_score)))))))
+
+(deftest format-output-test
+ (let [format-output (var-get #'curated-search/format-output)
+ ;; matches as produced by build-matches: :entity is already a full search-result record.
+ matches [{:saved_search_prompt "monthly revenue by region"
+ :usage_instructions "Use the Revenue metric; group by month."
+ :similarity 0.78 :weak? false
+ :score {:total_score 0.95}
+ :entity {:type "metric" :id 9 :name "Revenue" :database_id 1
+ :portable_entity_id "abcDEF"}}
+ {:saved_search_prompt "orders joined to customers"
+ :usage_instructions ""
+ :similarity 0.61 :weak? false
+ :score {:total_score 0.80}
+ :entity {:type "table" :id 1 :name "ORDERS" :database_schema "PUBLIC"
+ :database_id 1}}]
+ out (format-output matches)]
+ (testing "renders score, similarity, confidence, the saved prompt, and hydrated entity details"
+ (is (str/includes? out "score=\"0.950\""))
+ (is (str/includes? out "similarity=\"0.780\""))
+ (is (str/includes? out "confidence=\"strong\""))
+ (is (str/includes? out "<saved_search_prompt>monthly revenue by region</saved_search_prompt>"))
+ (is (str/includes? out "Revenue"))
+ (is (str/includes? out "ORDERS")))
+ (testing "usage instructions surface when present and are omitted when blank"
+ (is (str/includes? out "<usage_instructions>Use the Revenue metric; group by month.</usage_instructions>"))
+ (is (= 1 (count (re-seq #"<usage_instructions>" out)))))
+ (testing "no leading weak <note> when the top match is strong"
+ (is (not (str/starts-with? out "<note>"))))))
+
+(deftest xml-escaping-test
+ (let [format-output (var-get #'curated-search/format-output)
+ matches [{:saved_search_prompt "P&L by quarter <2026>"
+ :usage_instructions "Profit & loss; values < 0 are losses."
+ :similarity 0.7 :weak? false
+ :score {:total_score 0.7}
+ :entity {:type "table" :id 1 :name "FINANCE" :database_id 1}}]
+ out (format-output matches)]
+ (testing "curator-entered text is escaped in the XML output"
+ (is (str/includes? out "<saved_search_prompt>P&amp;L by quarter &lt;2026&gt;</saved_search_prompt>"))
+ (is (str/includes? out "<usage_instructions>Profit &amp; loss; values &lt; 0 are losses.</usage_instructions>")))))
+
+(deftest weak-match-note-test
+ (let [format-output (var-get #'curated-search/format-output)
+ matches [{:saved_search_prompt "something only loosely related"
+ :usage_instructions "" :similarity 0.20 :weak? true
+ :score {:total_score 0.20}
+ :entity {:type "table" :id 7 :name "MISC" :database_id 1}}]
+ out (format-output matches)]
+ (testing "a weak top match prepends the cautionary note and marks confidence weak"
+ (is (str/starts-with? out "<note>"))
+ (is (str/includes? out "No strong curated match"))
+ (is (str/includes? out "confidence=\"weak\"")))))
+
+(deftest flatten-data-test
+ (let [flatten-data (var-get #'curated-search/flatten-data)
+ matches [{:saved_search_prompt "p1" :usage_instructions "u1"
+ :score {:total_score 0.9} :similarity 0.7 :weak? false
+ :entity {:type "metric" :id 9 :name "Revenue"}}
+ {:saved_search_prompt "p2" :usage_instructions "u2"
+ :score {:total_score 0.6} :similarity 0.3 :weak? true
+ :entity {:type "table" :id 1 :name "ORDERS"}}]
+ out (flatten-data matches)]
+ (testing "one annotated record per match"
+ (is (= 2 (count out)))
+ (is (=? [{:type "metric" :id 9 :saved_search_prompt "p1" :usage_instructions "u1"
+ :confidence "strong" :similarity 0.7}
+ {:type "table" :id 1 :saved_search_prompt "p2" :usage_instructions "u2"
+ :confidence "weak" :similarity 0.3}]
+ out)))))
diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.clj
index 70ff10d75a8..edefca12e38 100644
--- a/test/metabase/metabot/tools/search_test.clj
+++ b/test/metabase/metabot/tools/search_test.clj
@@ -422,6 +422,66 @@
(is (some? dash-res) "expected the dashboard to appear in search results")
(is (not (contains? dash-res :portable_entity_id))))))))))
+(deftest entity-refs->search-results-test
+ (testing "hydrates {:model :id} refs (as stored by the semantic layer) into enriched search records"
+ (mt/with-test-user :crowberto
+ (mt/with-temp [:model/Card {m-id :id m-eid :entity_id}
+ {:name "Hydrate Sample Model" :type :model
+ :database_id (mt/id) :table_id (mt/id :orders)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (mt/id :orders)}}}]
+ (mt/with-temp [:model/Card {q-id :id} {:name "Hydrate Sample Question"
+ :database_id (mt/id) :table_id (mt/id :orders)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (mt/id :orders)}}}]
+ (let [results (search/entity-refs->search-results
+ [{:model "model" :id m-id}
+ {:model "table" :id (mt/id :orders)}
+ {:model "card" :id q-id} ; normalized to "question"
+ {:model "model" :id Integer/MAX_VALUE}]) ; nonexistent → dropped
+ by-id (into {} (map (juxt (juxt :type :id) identity)) results)]
+ (testing "model ref hydrates with type, name, and portable_entity_id"
+ (is (=? {:type "model" :name "Hydrate Sample Model" :portable_entity_id m-eid
+ :database_id (mt/id)}
+ (get by-id ["model" m-id]))))
+ (testing "table ref hydrates with type table and a database name"
+ (is (=? {:type "table" :database_id (mt/id) :database_name string?}
+ (get by-id ["table" (mt/id :orders)]))))
+ (testing "a card ref hydrates as the agent-facing type question"
+ (is (=? {:type "question" :name "Hydrate Sample Question"}
+ (get by-id ["question" q-id]))))
+ (testing "refs whose entity no longer exists are dropped"
+ (is (= 3 (count results))))))))))
+
+(deftest entity-refs->search-results-same-card-two-types-test
+ (testing "a card referenced under two type strings hydrates to one record per ref (neither is dropped)"
+ (mt/with-test-user :crowberto
+ (mt/with-temp [:model/Card {c-id :id} {:name "Dual Typed"
+ :database_id (mt/id) :table_id (mt/id :orders)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (mt/id :orders)}}}]
+ (let [results (search/entity-refs->search-results
+ [{:model "model" :id c-id} {:model "metric" :id c-id}])]
+ (is (= #{["model" c-id] ["metric" c-id]}
+ (set (map (juxt :type :id) results)))))))))
+
+(deftest entity-refs->search-results-respects-read-perms-test
+ (testing "hydration drops entities the current user can't read — a curated entry may point at a restricted one"
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Collection {coll-id :id} {}
+ :model/Card {restricted-id :id}
+ {:name "Secret Card" :collection_id coll-id
+ :database_id (mt/id) :table_id (mt/id :orders)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (mt/id :orders)}}}]
+ (let [refs [{:model "card" :id restricted-id}]]
+ (testing "a superuser can read it"
+ (mt/with-test-user :crowberto
+ (is (= [restricted-id] (map :id (search/entity-refs->search-results refs))))))
+ (testing "a user without access to its collection does not see it"
+ (mt/with-test-user :rasta
+ (is (empty? (search/entity-refs->search-results refs))))))))))
+
(deftest enrich-with-metric-base-tables-test
(testing (str "Metric search results carry `base_table_*` fields so the LLM can write\n"
"`source-table:` without a separate entity_details call. We look up\n"
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 4634ef98c9a..d77b8a4e3da 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -67,12 +67,17 @@
(is (contains? tools "read_resource"))
(is (contains? tools "ask_for_sql_clarification"))))
-(deftest ^:parallel get-tools-for-nlq-profile-test
- (let [tools (tools-for-profile :nlq)]
- (is (map? tools))
- (is (contains? tools "search"))
- (is (contains? tools "construct_notebook_query"))
- (is (contains? tools "create_chart"))))
+(deftest get-tools-for-nlq-profile-test
+ (testing "the general search is always available; the curated search tool only with the semantic-search feature"
+ (mt/with-premium-features #{}
+ (let [tools (tools-for-profile :nlq)]
+ (is (map? tools))
+ (is (contains? tools "search"))
+ (is (contains? tools "construct_notebook_query"))
+ (is (contains? tools "create_chart"))
+ (is (not (contains? tools "search_curated")))))
+ (mt/with-premium-features #{:semantic-search}
+ (is (contains? (tools-for-profile :nlq) "search_curated")))))
(deftest ^:parallel get-tools-for-document-generate-content-profile-test
(let [tools (tools-for-profile :document-generate-content)]