The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add search_curated tool for NLQ (#75237)
Chris Truter0bc758804b80
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 92743f522fb..58d115d4441 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -444,6 +444,7 @@instaparse.core/defparser clojure.core/defmetabase-enterprise.action-v2.test-util/with-test-tables! clojure.core/letmetabase-enterprise.content-translation.routes-test/with-new-secret-key! clojure.core/fn+ metabase-enterprise.curated-search.reconcile-test/with-isolated-mirror clojure.core/fnmetabase-enterprise.serialization.metadata-file-import.wire-format-test/with-tiny-fixture! clojure.core/fnmetabase-enterprise.serialization.test-util/with-dbs clojure.core/fnmetabase-enterprise.serialization.test-util/with-random-dump-dir clojure.core/let@@ -460,6 +461,7 @@metabase.channel.impl.http-test/with-captured-http-requests clojure.core/fnmetabase.channel.impl.http-test/with-server clojure.core/letmetabase.collections.models.collection-test/with-collection-in-location clojure.core/let+ metabase.curated-search.api-test/with-test-entry clojure.core/letmetabase.driver.mongo.connection/with-mongo-client clojure.core/letmetabase.driver.mongo.connection/with-mongo-database clojure.core/letmetabase.driver.mongo.query-processor/mongo-let clojure.core/letdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 273ceace815..034e1335935 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -682,6 +682,18 @@:api #{}:uses :any}+ curated-search+ {:team "Metabot"+ :api #{metabase.curated-search.api+ metabase.curated-search.core}+ :uses #{api+ models+ premium-features+ request+ util}+ :model-exports #{:model/CuratedSearchEntry}+ :model-imports #{:model/Card}}+dashboards{:team "UX West":api #{metabase.dashboards.autoplace@@ -1263,6 +1275,7 @@channelcollectionsconfig+ curated-searchdocumentsdriverinterestingness@@ -2951,6 +2964,15 @@:api #{metabase-enterprise.core.init}:uses :any}+ enterprise/curated-search+ {:team "Metabot"+ :api #{metabase-enterprise.curated-search.init}+ :uses #{premium-features+ task+ util+ enterprise/semantic-search}+ :model-imports #{:model/CuratedSearchEntry}}+enterprise/custom-viz-plugin{:team "Gadget":api #{metabase-enterprise.custom-viz-plugin.api@@ -3394,6 +3416,8 @@{:team "Metabot":api #{metabase-enterprise.semantic-search.apimetabase-enterprise.semantic-search.core+ metabase-enterprise.semantic-search.db.datasource+ metabase-enterprise.semantic-search.embeddingmetabase-enterprise.semantic-search.init}:uses #{activity-feedanalyticsdiff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.ymlindex 4eaa04f99f2..d0dbcda1688 100644--- a/.github/workflows/semantic-search.yml+++ b/.github/workflows/semantic-search.yml@@ -27,7 +27,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -86,7 +86,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -159,7 +159,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -232,7 +232,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/curated_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex c441cf49ea9..227749a7ad8 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -10,6 +10,7 @@[metabase-enterprise.advanced-config.init][metabase-enterprise.audit-app.init][metabase-enterprise.cache.init]+ [metabase-enterprise.curated-search.init][metabase-enterprise.custom-viz-plugin.init][metabase-enterprise.data-complexity-score.init][metabase-enterprise.database-replication.init]diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/core.clj b/enterprise/backend/src/metabase_enterprise/curated_search/core.cljnew file mode 100644index 00000000000..d8cdadb31b5--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/curated_search/core.clj@@ -0,0 +1,120 @@+(ns metabase-enterprise.curated-search.core+ "Enterprise implementation of the curated search mirror: pgvector-backed similarity search over the+ curated `curated_search_entries` table, and the write-path nudge that keeps the mirror fresh.++ OSS shims live in [[metabase.curated-search.mirror]]; the background sync they nudge is the+ [[metabase-enterprise.curated-search.task.sync]] Quartz job."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [honey.sql :as sql]+ [honey.sql.helpers :as sql.helpers]+ [metabase-enterprise.curated-search.index-table :as index-table]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedding :as embedding]+ [metabase.premium-features.core :as premium-features :refer [defenterprise]]+ [metabase.task.core :as task]+ [metabase.util.json :as json]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs])+ (:import+ (java.sql SQLException)+ (org.postgresql.util PGobject)))++(set! *warn-on-reflection* true)++(def sync-job-key+ "Quartz job key of the background sync; [[request-sync!]] triggers it and+ [[metabase-enterprise.curated-search.task.sync]] schedules it."+ (jobs/key "metabase-enterprise.curated-search.sync.job"))++(defn pgvector-configured?+ "True when a pgvector store is configured (read once at boot from MB_PGVECTOR_DB_URL).+ The sync task gates scheduling on this rather than [[available?]], so the periodic safety net exists+ even when the semantic-search feature is turned on after startup (a common onboarding flow)."+ []+ (string? (not-empty semantic.db.datasource/db-url)))++(defn available?+ "Whether the curated search mirror can run right now: a pgvector store is configured and the license+ includes semantic search.+ The feature check can flip at runtime (token entered post-boot), so callers re-evaluate per use."+ []+ (and (pgvector-configured?)+ (premium-features/has-feature? :semantic-search)))++(defenterprise request-sync!+ "Trigger the background sync job to reconcile the mirror soon.+ Fire-and-forget: [[metabase.task.core/trigger-now!]] swallows scheduler errors, and the job's periodic+ schedule covers anything a lost trigger misses."+ :feature :semantic-search+ []+ (when (available?)+ (task/trigger-now! sync-job-key)))++;; Weighted scoring shaped like regular search (metabase.search.scoring): each factor contributes+;; weight * score; similarity (1 - cosine distance) is primary, verified a flat 0/1 boost.+(def ^:private similarity-weight 1.0)+(def ^:private verified-weight 0.1)+(def ^:private default-limit 10)++(defn- scorer [nm score weight]+ (let [score (double score) weight (double weight)]+ {:name nm :score score :weight weight :contribution (* score weight)}))++(defn- score+ "Build a weighted-scorer breakdown for one row, shaped like regular search's scoring.+ Returns a `:scores` vector of `{:name :score :weight :contribution}` plus the weighted-sum+ `:total_score`."+ [{:keys [distance verified]}]+ (let [scores [(scorer :similarity (- 1.0 (double distance)) similarity-weight)+ (scorer :verified (if verified 1.0 0.0) verified-weight)]]+ {:scores scores+ :total_score (reduce + (map :contribution scores))}))++(defn- decode-entity [v]+ (cond+ (instance? PGobject v) (json/decode (.getValue ^PGobject v) true)+ (string? v) (json/decode v true)+ :else v))++(defenterprise search+ "Find the saved prompts nearest to `user-search-prompt` by cosine distance, up to `limit`.+ Results are ranked by blended score (similarity + verified boost), each shaped+ `{:saved_search_prompt :usage_instructions :entity :score}`.+ Returns [] when the pgvector store is unconfigured."+ :feature :semantic-search+ [user-search-prompt limit]+ (if-not (available?)+ []+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)+ limit (or limit default-limit)+ model (embedding/get-configured-model)+ embedding (embedding/get-embedding model user-search-prompt+ {:type :query :record-tokens? true})+ lit (index-table/format-embedding embedding)+ ;; Order by the blended score, not raw distance, so a verified row whose boost lifts it into+ ;; the top N survives the LIMIT. No HNSW index could accelerate this expression — see+ ;; index-table for why the table has none.+ ranking (format "(embedding <=> %s) - (CASE WHEN verified THEN %s ELSE 0.0 END)"+ lit verified-weight)+ rows (try+ (jdbc/execute!+ pgvector+ (-> (sql.helpers/select :search_prompt :usage_instructions :entity :verified+ [[:raw (str "embedding <=> " lit)] :distance])+ (sql.helpers/from (keyword index-table/*vectors-table*))+ (sql.helpers/order-by [[:raw ranking] :asc])+ (sql.helpers/limit limit)+ (sql/format {:quoted true}))+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ ;; 42P01 = undefined table: the background sync hasn't created the mirror yet.+ (catch SQLException e+ (if (= "42P01" (.getSQLState e)) [] (throw e))))]+ (->> rows+ (map (fn [row]+ {:saved_search_prompt (:search_prompt row)+ :usage_instructions (:usage_instructions row)+ :entity (decode-entity (:entity row))+ :score (score row)}))+ (sort-by (comp :total_score :score) >)+ vec))))diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/index_table.clj b/enterprise/backend/src/metabase_enterprise/curated_search/index_table.cljnew file mode 100644index 00000000000..508c90c0a04--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/curated_search/index_table.clj@@ -0,0 +1,144 @@+(ns metabase-enterprise.curated-search.index-table+ "Lifecycle of the pgvector tables backing the curated search mirror.++ Two tables live in the pgvector store:++ - the vectors table: one row per appdb `curated_search_entries` row, carrying the embedding, a copy of+ the searchable fields, and a `content_hash` used by the reconciler to detect stale rows;+ - a single-row meta table recording the embedding model identity and schema version the vectors table+ was built for.++ There is deliberately no HNSW index: the search path orders by the blended score (distance minus the+ verified boost), which an HNSW index can't accelerate anyway, and an exact scan over a curated table+ of this size is microseconds.++ [[ensure-tables!]] is the only entry point: it idempotently creates both tables and, when the+ configured embedding model or the schema version no longer matches the meta row, drops and recreates+ the vectors table so the next reconcile re-embeds everything.+ That drop-and-rebuild is the entire model-change story — the appdb table is authoritative and small,+ so rebuilding from it is cheap."+ (:require+ [clojure.string :as str]+ [honey.sql :as sql]+ [honey.sql.helpers :as sql.helpers]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs])+ (:import+ (java.time Instant)))++(set! *warn-on-reflection* true)++(def ^:dynamic *vectors-table*+ "Vectors table name. Dynamic so tests can rebind it to an isolated table."+ "curated_search_index")++(def ^:dynamic *meta-table*+ "Meta table name. Dynamic so tests can rebind it to an isolated table."+ "curated_search_index_meta")++(def schema-version+ "Version of the vectors table schema.+ Bump it to force a drop-and-rebuild on instances whose meta row records an older version."+ ;; 2: dropped the HNSW index — the rebuild sheds it from tables built under version 1.+ 2)++;; Advisory lock serializing concurrent ensure-tables! calls (e.g. several cluster nodes starting at+;; once). Arbitrary app-wide-unique constant; semantic-search's migration lock uses 19991.+(def ^:private ensure-lock-id 20011)++(defn- sql-format-quoted [honey-sql]+ (sql/format honey-sql {:quoted true}))++(defn format-embedding+ "Format a float-array embedding as a pgvector SQL literal.+ Throws if any element is not a finite number."+ [embedding]+ (doseq [v embedding]+ ;; NaN/Infinity satisfy `number?` but render as SQL literals pgvector chokes on, so require finite.+ (when-not (and (number? v) (Double/isFinite (double v)))+ (throw (ex-info "Embedding contains invalid value" {:invalid-value v}))))+ (str "'[" (str/join ", " embedding) "]'::vector"))++(defn- create-meta-table-sql []+ (-> (sql.helpers/create-table (keyword *meta-table*) :if-not-exists)+ (sql.helpers/with-columns+ [[:id :smallint [:primary-key] [:default 1] [:check [:= :id 1]]]+ [:provider :text :not-null]+ [:model_name :text :not-null]+ [:vector_dimensions :int :not-null]+ [:schema_version :int :not-null]+ [:updated_at :timestamp-with-time-zone :not-null]])+ sql-format-quoted))++(defn- create-vectors-table-sql [dims]+ (-> (sql.helpers/create-table (keyword *vectors-table*) :if-not-exists)+ (sql.helpers/with-columns+ [[:index_id :bigint [:primary-key]]+ [:search_prompt :text :not-null]+ [:usage_instructions :text :not-null]+ [:entity :jsonb :not-null]+ [:verified :boolean [:default false] :not-null]+ [:content_hash :text :not-null]+ [:embedding [:raw (format "vector(%d)" dims)] :not-null]])+ sql-format-quoted))++(defn- model-identity+ "The part of the meta row that identifies what the vectors table was built for."+ [embedding-model]+ {:provider (:provider embedding-model)+ :model_name (:model-name embedding-model)+ :vector_dimensions (:vector-dimensions embedding-model)+ :schema_version schema-version})++(defn- read-meta [tx]+ (jdbc/execute-one! tx [(format "SELECT provider, model_name, vector_dimensions, schema_version FROM \"%s\" WHERE id = 1" *meta-table*)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))++(defn- write-meta! [tx embedding-model]+ (let [{:keys [provider model_name vector_dimensions schema_version]} (model-identity embedding-model)]+ (jdbc/execute! tx+ (-> (sql.helpers/insert-into (keyword *meta-table*))+ (sql.helpers/values [{:id 1+ :provider provider+ :model_name model_name+ :vector_dimensions vector_dimensions+ :schema_version schema_version+ :updated_at (Instant/now)}])+ (sql.helpers/on-conflict :id)+ (sql.helpers/do-update-set :provider :model_name :vector_dimensions+ :schema_version :updated_at)+ sql-format-quoted))))++(defn- create-tables! [tx dims]+ (jdbc/execute! tx (create-vectors-table-sql dims)))++(defn ensure-tables!+ "Idempotently create the vectors + meta tables for `embedding-model`, rebuilding on model change.++ When the meta row's model identity (provider, model name, dimensions) or schema version no longer+ matches, the vectors table is dropped and recreated empty — the next reconcile re-embeds every row.+ Serialized across nodes with an advisory lock. Returns :created, :rebuilt or :ok."+ [pgvector embedding-model]+ (jdbc/with-transaction [tx pgvector]+ (jdbc/execute! tx [(format "SELECT pg_advisory_xact_lock(%d)" ensure-lock-id)])+ (jdbc/execute! tx (sql/format (sql.helpers/create-extension :vector :if-not-exists)))+ (jdbc/execute! tx (create-meta-table-sql))+ (let [stored (read-meta tx)+ current (model-identity embedding-model)+ dims (:vector_dimensions current)]+ (cond+ (nil? stored)+ (do (create-tables! tx dims)+ (write-meta! tx embedding-model)+ :created)++ (not= stored current)+ (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS \"%s\"" *vectors-table*)])+ (create-tables! tx dims)+ (write-meta! tx embedding-model)+ :rebuilt)++ :else+ ;; Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table heals itself.+ (do (create-tables! tx dims)+ :ok)))))diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/init.clj b/enterprise/backend/src/metabase_enterprise/curated_search/init.cljnew file mode 100644index 00000000000..17174e85500--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/curated_search/init.clj@@ -0,0 +1,5 @@+(ns metabase-enterprise.curated-search.init+ "Loads the curated-search module's side-effecting namespaces (the background sync task) at+ system startup. See [[metabase-enterprise.core.init]]."+ (:require+ [metabase-enterprise.curated-search.task.sync]))diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.clj b/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.cljnew file mode 100644index 00000000000..c85f4404eaa--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/curated_search/reconcile.clj@@ -0,0 +1,112 @@+(ns metabase-enterprise.curated-search.reconcile+ "Reconciles the pgvector mirror with the authoritative appdb `curated_search_entries` table.++ Rather than streaming individual writes (the table is curated and small — hundreds to low thousands of+ rows), [[reconcile!]] diffs the whole table against the mirror by content hash on every run: rows whose+ hash is missing or stale are re-embedded and upserted, mirror rows with no appdb counterpart are+ deleted, unchanged rows cost nothing.+ Because each run re-derives the mirror from the appdb, any missed write — pgvector downtime, a crashed+ node, an import that bypassed the model hooks — is repaired on the next run with no operator action.++ Runs from the [[metabase-enterprise.curated-search.task.sync]] Quartz job; callers supply the+ datasource and embedding model so this namespace reads no settings itself."+ (:require+ [buddy.core.hash :as buddy-hash]+ [honey.sql :as sql]+ [honey.sql.helpers :as sql.helpers]+ [metabase-enterprise.curated-search.index-table :as index-table]+ [metabase-enterprise.semantic-search.embedding :as embedding]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private embed-batch-size+ "Rows embedded per embedding-service request while backfilling stale mirror rows."+ 50)++(defn content-hash+ "Hash of an appdb row's mirror-relevant fields.+ Stored on the mirror row at upsert time; a mismatch (or absence) marks the row stale."+ [{:keys [search_prompt usage_instructions entity verified]}]+ (u/encode-base64-bytes+ (buddy-hash/sha1+ (json/encode (sorted-map :search_prompt search_prompt+ :usage_instructions (or usage_instructions "")+ :entity entity+ :verified (boolean verified))))))++(defn- mirror-hashes+ "Map of index_id -> content_hash for every row currently in the mirror."+ [pgvector]+ (into {}+ (map (juxt :index_id :content_hash))+ (jdbc/execute! pgvector+ [(format "SELECT index_id, content_hash FROM \"%s\"" index-table/*vectors-table*)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))++(defn- delete-rows! [pgvector ids]+ (when (seq ids)+ (jdbc/execute! pgvector+ (-> (sql.helpers/delete-from (keyword index-table/*vectors-table*))+ (sql.helpers/where [:in :index_id ids])+ (sql/format {:quoted true})))))++(defn- row->record [row hash embedding]+ {:index_id (:id row)+ :search_prompt (:search_prompt row)+ :usage_instructions (or (:usage_instructions row) "")+ :entity [:cast (json/encode (:entity row)) :jsonb]+ :verified (boolean (:verified row))+ :content_hash hash+ :embedding [:raw (index-table/format-embedding embedding)]})++(defn- upsert-batch!+ "Embed one batch of stale appdb rows and upsert them into the mirror.+ Returns the number of rows upserted."+ [pgvector embedding-model rows]+ (let [embeddings (embedding/get-embeddings-batch embedding-model (map :search_prompt rows)+ {:type :index :record-tokens? true})+ records (map (fn [row embedding] (row->record row (content-hash row) embedding))+ rows embeddings)]+ (jdbc/execute! pgvector+ (-> (sql.helpers/insert-into (keyword index-table/*vectors-table*))+ (sql.helpers/values (vec records))+ (sql.helpers/on-conflict :index_id)+ (sql.helpers/do-update-set :search_prompt :usage_instructions :entity+ :verified :content_hash :embedding)+ (sql/format {:quoted true})))+ (count rows)))++(defn reconcile!+ "Bring the pgvector mirror in line with the appdb table; see the namespace docstring for the approach.+ Embedding-service calls are scoped to changed rows only.+ A failed batch is logged and skipped — its rows stay stale and are retried on the next run.+ Returns {:upserted n :deleted n :unchanged n}."+ [pgvector embedding-model]+ (index-table/ensure-tables! pgvector embedding-model)+ (let [appdb-rows (t2/select [:model/CuratedSearchEntry :id :search_prompt :usage_instructions+ :entity :verified])+ mirrored (mirror-hashes pgvector)+ stale (remove #(= (content-hash %) (get mirrored (:id %))) appdb-rows)+ orphans (remove (set (map :id appdb-rows)) (keys mirrored))+ upserted (transduce+ (partition-all embed-batch-size)+ (completing+ (fn [n batch]+ (+ n (try+ (upsert-batch! pgvector embedding-model batch)+ (catch Exception e+ (log/error e "curated search mirror: failed to upsert batch of"+ (count batch) "rows; will retry next run")+ 0)))))+ 0+ stale)]+ (delete-rows! pgvector orphans)+ {:upserted upserted+ :deleted (count orphans)+ :unchanged (- (count appdb-rows) (count stale))}))diff --git a/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.clj b/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.cljnew file mode 100644index 00000000000..2829ff740c1--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/curated_search/task/sync.clj@@ -0,0 +1,59 @@+(ns metabase-enterprise.curated-search.task.sync+ "Quartz job running the curated search mirror reconciliation.++ Scheduled periodically as a safety net; appdb writes also trigger it immediately via+ [[metabase-enterprise.curated-search.core/request-sync!]], so curator edits become searchable+ within seconds.+ `DisallowConcurrentExecution` plus Quartz's trigger coalescing debounce edit bursts to at most one+ queued extra run, and Quartz clustering ensures a single node runs it."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.simple :as simple]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase-enterprise.curated-search.core :as curated-search.core]+ [metabase-enterprise.curated-search.reconcile :as reconcile]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedding :as embedding]+ [metabase.task.core :as task]+ [metabase.util.log :as log])+ (:import+ (java.time Duration Instant)+ (java.util Date)))++(set! *warn-on-reflection* true)++(task/defjob ^{org.quartz.DisallowConcurrentExecution true+ :doc "Reconciles the curated search pgvector mirror with the appdb table."}+ CuratedSearchEntrySync [_ctx]+ (when (curated-search.core/available?)+ (try+ (let [result (reconcile/reconcile! (semantic.db.datasource/ensure-initialized-data-source!)+ (embedding/get-configured-model))]+ (when (pos? (+ (:upserted result) (:deleted result)))+ (log/info "curated search mirror reconciled" result)))+ (catch Throwable e+ ;; Log and move on: the next periodic run retries from the authoritative appdb table.+ (log/error e "curated search mirror reconciliation failed")))))++(def ^:private ^Duration startup-delay (Duration/parse "PT10S"))+(def ^:private ^Duration run-frequency (Duration/parse "PT30S"))++(defmethod task/init! ::CuratedSearchEntrySync [_]+ ;; Gate scheduling on pgvector being configured (boot-fixed), NOT on available? — the job body+ ;; self-gates on the feature flag, so scheduling here lets the periodic safety net (and the+ ;; write-path trigger's target job) exist even when the license is enabled after startup.+ (when (curated-search.core/pgvector-configured?)+ (let [job (jobs/build+ (jobs/of-type CuratedSearchEntrySync)+ (jobs/store-durably)+ (jobs/with-identity curated-search.core/sync-job-key))+ trigger (triggers/build+ (triggers/with-identity+ (triggers/key "metabase-enterprise.curated-search.sync.trigger"))+ (triggers/for-job curated-search.core/sync-job-key)+ (triggers/start-at (Date/from (.plus (Instant/now) startup-delay)))+ (triggers/with-schedule+ (simple/schedule+ (simple/with-interval-in-milliseconds (.toMillis run-frequency))+ (simple/repeat-forever))))]+ (task/schedule-task! job trigger))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 700fcdd57a0..120114582d6 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -42,7 +42,10 @@(conj "EmbeddingTheme")(not (:no-custom-viz-plugins opts))- (conj "CustomVizPlugin")))+ (conj "CustomVizPlugin")++ (not (:no-curated-search opts))+ (conj "CuratedSearchEntry")))(defn make-targets-of-type"Returns a targets seq with model type and given ids"diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.cljindex 8a41f1bd9c7..c90393d4394 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj@@ -82,7 +82,7 @@(def legal-top-level-paths"Known top-level paths for directory with serialization output.We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."- #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "snippets" "transforms"})+ #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "curated_search_entries" "snippets" "transforms"})(defn- ingest-all"Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 17aa985bb26..d18257ffbd3 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -33,7 +33,8 @@"The list of all models exported by serialization by default. Used for production code and by tests."(concat data-modelcontent- ["CustomVizPlugin"+ ["CuratedSearchEntry"+ "CustomVizPlugin""EmbeddingTheme""FieldValues""Metabot"diff --git a/enterprise/backend/test/metabase_enterprise/curated_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/curated_search/core_test.cljnew file mode 100644index 00000000000..2138f07a394--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/curated_search/core_test.clj@@ -0,0 +1,150 @@+(ns metabase-enterprise.curated-search.core-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.curated-search.core :as curated-search.core]+ [metabase-enterprise.curated-search.index-table :as index-table]+ [metabase-enterprise.curated-search.reconcile :as reconcile]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.curated-search.mirror :as mirror]+ [metabase.metabot.tools.curated-search :as tools.curated-search]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]))++(set! *warn-on-reflection* true)++(defn- approx [target] #(< (abs (- (double %) (double target))) 1e-9))++(deftest score-shape-matches-regular-search-test+ (let [score (var-get #'curated-search.core/score)]+ (testing "weighted-scorer breakdown: per-factor {name score weight contribution} + total_score"+ (is (=? {:scores [{:name :similarity :score (approx 0.8) :weight 1.0 :contribution (approx 0.8)}+ {:name :verified :score 0.0 :weight 0.1 :contribution 0.0}]+ :total_score (approx 0.8)}+ (score {:distance 0.2 :verified false}))))+ (testing "the verified boost applies its weight and strictly increases the total"+ (is (=? {:total_score (approx 0.9)} (score {:distance 0.2 :verified true})))+ (let [s (score {:distance 0.2 :verified true})]+ (is (=? (approx (reduce + (map :contribution (:scores s)))) (:total_score s))))+ (is (> (:total_score (score {:distance 0.2 :verified true}))+ (:total_score (score {:distance 0.2 :verified false})))))))++(deftest dispatch-without-pgvector-test+ (testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"+ ;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:+ ;; available? is false, so search returns [] and the sync nudge no-ops rather than throwing.+ (mt/with-premium-features #{:semantic-search}+ (with-redefs [semantic.db.datasource/db-url nil]+ (is (= [] (mirror/search "anything" 10)))+ (is (nil? (mirror/request-sync!)))))))++(deftest request-sync-triggers-the-job-test+ (testing "when the mirror is available, the nudge fires trigger-now! on the sync job — and nothing else"+ (mt/with-premium-features #{:semantic-search}+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]+ (let [triggered (atom [])]+ (mt/with-dynamic-fn-redefs [task/trigger-now! (fn [job-key] (swap! triggered conj job-key))]+ (mirror/request-sync!)+ (is (= [curated-search.core/sync-job-key] @triggered))))))))++(deftest pgvector-configured-decoupled-from-feature-test+ (testing "scheduling gates on pgvector config, independent of the feature flag, so a license enabled after boot still gets the periodic sync"+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]+ (mt/with-premium-features #{}+ (is (true? (curated-search.core/pgvector-configured?)))+ (is (false? (curated-search.core/available?))))+ (mt/with-premium-features #{:semantic-search}+ (is (true? (curated-search.core/available?)))))))++(deftest ^:sequential verified-boost-applies-before-limit-test+ (testing "the SQL LIMIT ranks by blended score: a verified row slightly farther by raw distance still wins top-1"+ (when semantic.db.datasource/db-url+ (let [suffix (System/nanoTime)+ ds (semantic.db.datasource/ensure-initialized-data-source!)+ p-verif "verified entry"+ p-plain "plain entry"+ q "the query"]+ (mt/with-premium-features #{:semantic-search}+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly semantic.tu/mock-embedding-model)]+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix)+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix)]+ ;; plain ties the query exactly (distance 0); verified sits at distance ~0.006, within its+ ;; 0.1 boost. A raw-distance LIMIT 1 would keep only the plain row.+ (semantic.tu/with-mock-embeddings {q [1.0 0.0 0.0 0.0]+ p-plain [1.0 0.0 0.0 0.0]+ p-verif [0.9 0.1 0.0 0.0]}+ (mt/with-temp [:model/CuratedSearchEntry _+ {:search_prompt p-plain :entity {:model "table" :id 1}}+ :model/CuratedSearchEntry _+ {:search_prompt p-verif :verified true :entity {:model "table" :id 2}}]+ (try+ (reconcile/reconcile! ds semantic.tu/mock-embedding-model)+ (is (= [p-verif] (mapv :saved_search_prompt (mirror/search q 1))))+ (finally+ (jdbc/execute! ds [(str "DROP TABLE IF EXISTS "+ index-table/*vectors-table* ", "+ index-table/*meta-table*)]))))))))))))++(defn- create-entry!+ "POST an entry through the CRUD API; returns the created row's id."+ [search-prompt entity verified]+ (:id (mt/user-http-request :crowberto :post 200 "curated-search/"+ {:search_prompt search-prompt :entity entity :verified verified})))++(deftest ^:sequential crud-api-to-tool-end-to-end-test+ (testing "CRUD API write -> reconcile -> pgvector -> search_curated tool, end to end"+ ;; Self-gated on MB_PGVECTOR_DB_URL — CI without semantic-search infra skips this; locally with the+ ;; dev pgvector running it exercises the whole pipeline. Uses the mock embedding model (4-dim,+ ;; deterministic, no network) against isolated temp tables, with the reconciler run directly in+ ;; place of the background Quartz job.+ (when semantic.db.datasource/db-url+ (let [suffix (System/nanoTime)+ ds (semantic.db.datasource/ensure-initialized-data-source!)+ ent-1 {:model "table" :id (mt/id :orders)}+ ent-2 {:model "table" :id (mt/id :people)}+ ;; Both prompts + the query embed to the same vector, so cosine distance ties and the+ ;; verified boost (0.10) alone decides the ranking.+ vec1 [1.0 0.0 0.0 0.0]+ p-verif "revenue by region (verified)"+ p-plain "revenue by region (plain)"+ q "monthly revenue per region"+ ;; bind a user: the tool's hydration permission-filters via mi/can-read?, which needs+ ;; *current-user-id* (a superuser can read the test tables).+ search! #(mt/with-test-user :crowberto+ (get-in (tools.curated-search/curated-search-tool+ {:user_search_prompt q})+ [:structured-output :data]))]+ (mt/with-premium-features #{:semantic-search}+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly semantic.tu/mock-embedding-model)]+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix)+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix)]+ (semantic.tu/with-mock-embeddings {p-verif vec1 p-plain vec1 q vec1}+ (try+ (let [id-verif (create-entry! p-verif ent-1 true)+ id-plain (create-entry! p-plain ent-2 false)+ _ (reconcile/reconcile! ds semantic.tu/mock-embedding-model)+ results (search!)]+ (testing "both distinct entities mirrored and returned, the verified one boosted first"+ (is (= [p-verif p-plain] (mapv :saved_search_prompt results))))+ (testing "data is per-entity hydrated records; total_score reflects the boost, similarity=1.0 (vectors tie)"+ (is (=? [{:type "table" :id (mt/id :orders) :similarity (approx 1.0)+ :score {:total_score (approx 1.1)}}+ {:type "table" :id (mt/id :people) :similarity (approx 1.0)+ :score {:total_score (approx 1.0)}}]+ results)))+ (testing "deleting via the CRUD API + reconcile removes the row from search results"+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" id-verif))+ (is (=? {:deleted 1} (reconcile/reconcile! ds semantic.tu/mock-embedding-model)))+ (is (= [p-plain] (mapv :saved_search_prompt (search!)))))+ ;; clean up the appdb rows we created+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" id-plain)))+ (finally+ (jdbc/execute! ds [(str "DROP TABLE IF EXISTS "+ index-table/*vectors-table* ", "+ index-table/*meta-table*)])))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.cljnew file mode 100644index 00000000000..a4537b6291a--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/curated_search/reconcile_test.clj@@ -0,0 +1,119 @@+(ns metabase-enterprise.curated-search.reconcile-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.curated-search.index-table :as index-table]+ [metabase-enterprise.curated-search.reconcile :as reconcile]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(deftest content-hash-test+ (let [row {:search_prompt "p" :usage_instructions "u" :entity {:model "table" :id 1}+ :verified false}]+ (testing "equal rows hash equal; nil and absent usage_instructions hash like empty"+ (is (= (reconcile/content-hash row) (reconcile/content-hash row)))+ (is (= (reconcile/content-hash (assoc row :usage_instructions nil))+ (reconcile/content-hash (assoc row :usage_instructions "")))))+ (testing "every mirror-relevant field changes the hash"+ (let [h (reconcile/content-hash row)]+ (doseq [variant [(assoc row :search_prompt "p2")+ (assoc row :usage_instructions "u2")+ (assoc row :entity {:model "table" :id 2})+ (assoc row :verified true)]]+ (is (not= h (reconcile/content-hash variant)) (pr-str variant)))))))++(deftest format-embedding-rejects-invalid-values-test+ (testing "non-numbers, NaN and infinities are rejected before they reach a raw SQL literal"+ (doseq [bad [Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY "0.1"]]+ (is (thrown-with-msg? Exception #"invalid value"+ (index-table/format-embedding [0.1 bad 0.3]))+ (pr-str bad)))+ (is (string? (index-table/format-embedding [0.1 -0.2 0.3])))))++(defmacro ^:private with-isolated-mirror+ "Run `body` with isolated pgvector tables bound and dropped afterwards, binding `ds-sym` to the+ pgvector datasource. Skips the body entirely when no pgvector store is configured."+ [[ds-sym] & body]+ `(when semantic.db.datasource/db-url+ (let [suffix# (System/nanoTime)+ ~ds-sym (semantic.db.datasource/ensure-initialized-data-source!)]+ (binding [index-table/*vectors-table* (str "curated_search_index_test_" suffix#)+ index-table/*meta-table* (str "curated_search_index_meta_test_" suffix#)]+ (try+ ~@body+ (finally+ (jdbc/execute! ~ds-sym [(str "DROP TABLE IF EXISTS "+ index-table/*vectors-table* ", "+ index-table/*meta-table*)])))))))++(defn- mirror-rows [ds]+ (jdbc/execute! ds+ [(format "SELECT index_id, search_prompt, verified, content_hash FROM \"%s\" ORDER BY index_id"+ index-table/*vectors-table*)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))++(deftest ^:sequential reconcile-lifecycle-test+ (with-isolated-mirror [ds]+ (let [model semantic.tu/mock-embedding-model]+ (mt/with-temp [:model/CuratedSearchEntry {id-1 :id}+ {:search_prompt "monthly revenue"+ :entity {:model "table" :id 1}}+ :model/CuratedSearchEntry {id-2 :id}+ {:search_prompt "orders and customers" :verified true+ :entity {:model "table" :id 2}}]+ (let [appdb-total (t2/count :model/CuratedSearchEntry)]+ (testing "first run mirrors every appdb row"+ (is (=? {:upserted appdb-total :deleted 0 :unchanged 0}+ (reconcile/reconcile! ds model)))+ (is (=? {id-1 {:search_prompt "monthly revenue" :verified false}+ id-2 {:search_prompt "orders and customers" :verified true}}+ (-> (into {} (map (juxt :index_id identity)) (mirror-rows ds))+ (select-keys [id-1 id-2])))))+ (testing "an unchanged second run embeds and writes nothing"+ (is (=? {:upserted 0 :deleted 0 :unchanged appdb-total}+ (reconcile/reconcile! ds model))))+ (testing "an updated row is re-embedded; the rest stay unchanged"+ (t2/update! :model/CuratedSearchEntry id-1 {:verified true})+ (is (=? {:upserted 1 :deleted 0 :unchanged (dec appdb-total)}+ (reconcile/reconcile! ds model)))+ (is (=? {:verified true}+ (->> (mirror-rows ds) (filter #(= id-1 (:index_id %))) first))))+ (testing "a write that bypassed the hooks entirely is still repaired (self-healing)"+ ;; Tamper with the mirror behind the reconciler's back.+ (jdbc/execute! ds [(format "UPDATE \"%s\" SET content_hash = 'bogus' WHERE index_id = %d"+ index-table/*vectors-table* id-2)])+ (is (=? {:upserted 1 :deleted 0}+ (reconcile/reconcile! ds model))))))+ (testing "rows deleted from the appdb are removed from the mirror as orphans"+ ;; The with-temp rows above are gone now.+ (is (=? {:upserted 0 :deleted 2}+ (reconcile/reconcile! ds model)))))))++(deftest ^:sequential rebuild-on-model-change-test+ (with-isolated-mirror [ds]+ (let [model semantic.tu/mock-embedding-model]+ (mt/with-temp [:model/CuratedSearchEntry _+ {:search_prompt "monthly revenue"+ :entity {:model "table" :id 1}}]+ (let [appdb-total (t2/count :model/CuratedSearchEntry)]+ (testing "populate the mirror under the original model"+ (is (=? {:upserted appdb-total} (reconcile/reconcile! ds model))))+ (testing "a model-identity change drops the vectors table and the next run re-embeds everything"+ (let [new-model (assoc model :model-name "model-v2")]+ (is (= :rebuilt (index-table/ensure-tables! ds new-model)))+ (is (= [] (mirror-rows ds)))+ (is (=? {:upserted appdb-total :deleted 0 :unchanged 0}+ (reconcile/reconcile! ds new-model)))+ (testing "a schema-version mismatch alone also triggers the rebuild"+ ;; stale-write the meta row, as if the table were built by an older code version+ (jdbc/execute! ds [(format "UPDATE \"%s\" SET schema_version = schema_version - 1"+ index-table/*meta-table*)])+ (is (= :rebuilt (index-table/ensure-tables! ds new-model)))+ (is (= [] (mirror-rows ds))))+ (testing "the rebuild heals the meta row, so it doesn't recur on the next sync"+ (is (= :ok (index-table/ensure-tables! ds new-model)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.cljnew file mode 100644index 00000000000..c80fa683851--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/curated_search_entry_test.clj@@ -0,0 +1,136 @@+(ns metabase-enterprise.serialization.v2.curated-search-entry-test+ "Round-trip serialization tests for `:model/CuratedSearchEntry`, whose `entity` column holds a+ polymorphic Card/Table reference that must survive export and import."+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.test-util :as ts]+ [metabase-enterprise.serialization.v2.extract :as extract]+ [metabase-enterprise.serialization.v2.ingest :as serdes.ingest]+ [metabase-enterprise.serialization.v2.load :as serdes.load]+ [metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]+ [metabase.models.serialization :as serdes]+ [metabase.search.core :as search]+ [metabase.test :as mt]+ [metabase.warehouses.models.database :as models.database]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;; The references point at the H2 test-data Table, so keep the H2 guard off and re-enable H2 in extract.+#_{:clj-kondo/ignore [:metabase/validate-deftest]}+(use-fixtures :each (fn [thunk]+ (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)+ models.database/assert-not-h2! (constantly nil)]+ (binding [models.database/*include-h2-in-extract?* true]+ (thunk)))))++(defn- no-labels [path]+ (mapv #(dissoc % :label) path))++(defn- ingestion-in-memory [extractions]+ (let [mapped (into {} (for [entity (vec extractions)]+ [(no-labels (serdes/path entity)) entity]))]+ (reify serdes.ingest/Ingestable+ (ingest-list [_] (keys mapped))+ (ingest-one [_ path] (get mapped (no-labels path)))+ (ingest-errors [_] []))))++(deftest card-backed-entity-round-trip-test+ (testing "every card-backed entity model exports as the Card's entity_id and imports back"+ (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]+ (doseq [entity-model ["card" "model" "metric" "question"]]+ (testing entity-model+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id sli-eid :entity_id}+ {:search_prompt (str "find " entity-model) :verified true+ :usage_instructions "Use it directly."+ :entity {:model entity-model :id card-id}}]+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)]+ (testing "the entity ref is exported as a portable reference"+ (is (=? {:entity_id sli-eid+ :search_prompt (str "find " entity-model)+ :usage_instructions "Use it directly."+ :verified true+ :entity {:model entity-model :id card-eid}}+ extracted)))+ (testing "dependencies cover the referenced Card"+ (is (= #{[{:model "Card" :id card-eid}]}+ (serdes/dependencies extracted))))+ (testing "importing resolves the ref back to the local id"+ (t2/delete! :model/CuratedSearchEntry :id sli-id)+ (serdes.load/load-metabase! (ingestion-in-memory [extracted]))+ (is (= {:model entity-model :id card-id}+ (t2/select-one-fn :entity :model/CuratedSearchEntry :entity_id sli-eid)))+ ;; load-metabase! re-inserts under a new id that with-temp can't reap; clean it up so the+ ;; row doesn't leak into other tests' view of the (small, fully-scanned) table.+ (t2/delete! :model/CuratedSearchEntry :entity_id sli-eid)))))))))++(deftest unmapped-entity-model-passes-through-test+ (testing "an unrecognized entity model exports as-is (raw id, no deps) instead of aborting the export"+ ;; The CRUD API rejects unknown models, but rows written before a model string was retired (or by+ ;; direct appdb writes) shouldn't take down a whole export.+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id}+ {:search_prompt "legacy" :entity {:model "garbage" :id 5}}]+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)]+ (is (=? {:entity {:model "garbage" :id 5}} extracted))+ (is (= #{} (serdes/dependencies extracted)))))))++(deftest table-entity-round-trip-test+ (let [table-id (mt/id :venues)]+ (mt/with-temp [:model/CuratedSearchEntry {sli-id :id sli-eid :entity_id}+ {:search_prompt "best venues" :entity {:model "table" :id table-id}}]+ (let [extracted (ts/extract-one "CuratedSearchEntry" sli-id)+ table-ref (:entity extracted)]+ (testing "the table ref is exported as a [db schema table] path"+ (is (=? {:entity_id sli-eid+ :entity {:model "table" :id vector?}}+ extracted))+ (is (= ["VENUES"] (take-last 1 (:id table-ref)))))+ (testing "dependencies cover the Table's Database (the Table is synthesized on import)"+ (is (= #{[{:model "Database" :id (first (:id table-ref))}]}+ (serdes/dependencies extracted))))+ (testing "importing resolves the ref back to the local id"+ (t2/delete! :model/CuratedSearchEntry :id sli-id)+ (serdes.load/load-metabase! (ingestion-in-memory [extracted]))+ (is (= {:model "table" :id table-id}+ (t2/select-one-fn :entity :model/CuratedSearchEntry :entity_id sli-eid)))+ ;; load-metabase! re-inserts under a new id that with-temp can't reap; clean it up so the row+ ;; doesn't leak into other tests' view of the (small, fully-scanned) table.+ (t2/delete! :model/CuratedSearchEntry :entity_id sli-eid))))))++(deftest disk-export-import-round-trip-test+ (testing "CuratedSearchEntry survives a real on-disk export/import"+ ;; The in-memory tests above skip two layers that have silently dropped the model before:+ ;; `extract/model-set` (the default-export selection) and `ingest/legal-top-level-paths` (which+ ;; dirs the importer reads). This drives both through actual YAML on disk.+ (let [serialized (atom nil)+ sli-eid (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "Prompt DB")+ t1 (ts/create! :model/Table :name "ORDERS" :db_id (:id db) :schema "PUBLIC")+ sli (ts/create! :model/CuratedSearchEntry+ :search_prompt "orders" :verified true+ :entity {:model "table" :id (:id t1)})]+ (reset! sli-eid (:entity_id sli))+ ;; realize inside with-cache so the cached resolvers are actually used during extraction+ (reset! serialized (serdes/with-cache (into [] (extract/extract {}))))))+ (testing "default export selection includes it (guards extract/model-set)"+ (is (= 1 (count (filter (fn [{[{:keys [model]}] :serdes/meta}]+ (= model "CuratedSearchEntry"))+ @serialized)))))+ (ts/with-random-dump-dir [dump-dir "sli-serdes-"]+ (storage/store! (seq @serialized) (storage.files/file-writer dump-dir))+ (testing "stored under the curated_search_entries/ directory"+ (is (.exists (io/file dump-dir "curated_search_entries"))))+ (testing "ingest reads it back (guards ingest/legal-top-level-paths)"+ (is (contains? (set (serdes.ingest/ingest-list (serdes.ingest/ingest-yaml dump-dir)))+ [{:model "CuratedSearchEntry" :id @sli-eid}])))+ (testing "loads into a fresh appdb with the table ref resolved"+ (ts/with-db dest-db+ (serdes/with-cache (serdes.load/load-metabase! (serdes.ingest/ingest-yaml dump-dir)))+ (let [row (t2/select-one :model/CuratedSearchEntry :entity_id @sli-eid)]+ (is (=? {:verified true} row))+ (is (= "ORDERS"+ (t2/select-one-fn :name :model/Table :id (get-in row [:entity :id]))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex ea6085e1b92..5a496c3a32d 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -131,6 +131,12 @@(def ^:private internal-model?#{"Schema"})+(def ^:private covered-by-dedicated-round-trip-test?+ "Models that have full export/import coverage in their own round-trip test and so don't need a+ fixture in this shared baseline. CuratedSearchEntry is covered (in-memory + on-disk) by+ metabase-enterprise.serialization.v2.curated-search-entry-test."+ #{"CuratedSearchEntry"})+(defn add-to-baseline!"Use this within v2.extract-test where relevant to add their fixtures to the baseline."[]@@ -142,7 +148,7 @@resources (ingest/ingest-list ingestable)baselined (into #{} (map :model) (apply concat resources))necessary? (set serdes.models/exported-models)]- (doseq [m serdes.models/exported-models]+ (doseq [m serdes.models/exported-models :when (not (covered-by-dedicated-round-trip-test? m))](is (baselined m) (format "We need to add %s entries to %s" m source-dir-path)))(doseq [b baselined :when (not (internal-model? b))](is (necessary? b) (format "We can remove %s files from %s" b source-dir-path)))))diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex e15ce6f330a..6a2da90ca06 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -104,6 +104,7 @@collectionsconfigcontent-verification+ curated-searchcustom-viz-plugindashboardsdocumentsdiff --git a/resources/migrations/063/20260604_curated_search_entries.yaml b/resources/migrations/063/20260604_curated_search_entries.yamlnew file mode 100644index 00000000000..251d0f1e7ac--- /dev/null+++ b/resources/migrations/063/20260604_curated_search_entries.yaml@@ -0,0 +1,72 @@+## curated_search_entries: curated saved-search-prompt -> entity rows. The table is authoritative;+## an enterprise pgvector mirror (one embedding per row) backs the `curated_search` Metabot tool.++databaseChangeLog:+ - logicalFilePath: migrations/063_update_migrations.yaml++ - changeSet:+ id: v63.2026-06-04T00:00:00+ author: christruter+ comment: Create curated_search_entries table+ changes:+ - createTable:+ tableName: curated_search_entries+ remarks: Curated search prompts mapped to the entities that answer them+ columns:+ - column:+ name: id+ type: int+ remarks: Unique ID+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: search_prompt+ type: ${text.type}+ remarks: The saved search prompt text+ constraints:+ nullable: false+ - column:+ name: usage_instructions+ type: ${text.type}+ remarks: Agent-facing guidance on how to use the mapped entities+ constraints:+ nullable: true+ - column:+ name: entity+ type: ${text.type}+ remarks: JSON entity ref ({model, id, name?}) the prompt maps to+ constraints:+ nullable: false+ - column:+ name: verified+ type: ${boolean.type}+ remarks: Whether this prompt-entity mapping has been verified by a curator+ defaultValueBoolean: false+ constraints:+ nullable: false+ - column:+ name: entity_id+ type: char(21)+ remarks: Random NanoID tag for unique identity (serialization)+ constraints:+ nullable: true+ unique: true+ - column:+ name: created_at+ type: ${timestamp_type}+ remarks: When the mapping was created+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ - column:+ name: updated_at+ type: ${timestamp_type}+ remarks: When the mapping was last updated+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ rollback:+ - dropTable:+ tableName: curated_search_entriesdiff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex b9516e5e6e4..bce0045ed01 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -21,6 +21,7 @@[metabase.collections-rest.api][metabase.comments.api][metabase.config.core :as config]+ [metabase.curated-search.api][metabase.dashboards-rest.api][metabase.data-studio.api][metabase.documents.api]@@ -88,6 +89,7 @@metabase.cloud-migration.api/keep-memetabase.comments.api/keep-memetabase.collections-rest.api/keep-me+ metabase.curated-search.api/keep-memetabase.dashboards-rest.api/keep-memetabase.data-studio.api/keep-memetabase.documents.api/keep-me@@ -178,6 +180,7 @@"/cloud-migration" (+auth 'metabase.cloud-migration.api)"/collection" (+auth 'metabase.collections-rest.api)"/comment" (+auth metabase.comments.api/routes)+ "/curated-search" (+auth 'metabase.curated-search.api)"/dashboard" (+auth 'metabase.dashboards-rest.api)"/data-studio" (+auth metabase.data-studio.api/routes)"/database" (+auth 'metabase.warehouses-rest.api)diff --git a/src/metabase/cmd/copy.clj b/src/metabase/cmd/copy.cljindex 175f351ce60..e595a435cb2 100644--- a/src/metabase/cmd/copy.clj+++ b/src/metabase/cmd/copy.clj@@ -138,7 +138,8 @@:model/MetabotFeedback:model/MetabotSourceFeedback:model/MetabotUsedTable- :model/MetabotPrompt]+ :model/MetabotPrompt+ :model/CuratedSearchEntry](when config/ee-available?[:model/MetabotPermissions:model/MetabotGroupLimitdiff --git a/src/metabase/curated_search/api.clj b/src/metabase/curated_search/api.cljnew file mode 100644index 00000000000..01e1ff20032--- /dev/null+++ b/src/metabase/curated_search/api.clj@@ -0,0 +1,117 @@+(ns metabase.curated-search.api+ "Admin REST API for managing curated search entries: curated search prompts mapped to the entities+ that answer them."+ (:require+ [metabase.api.common :as api]+ [metabase.api.macros :as api.macros]+ [metabase.request.core :as request]+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2]))++(def ^:private EntityRef+ "Entity ref as accepted on writes: the model must be a known agent-facing entity type (as used with+ read_resource), plus plain \"card\"."+ [:map+ [:model [:enum "table" "card" "model" "metric" "question"]]+ [:id ms/PositiveInt]+ [:name {:optional true} :string]])++(def ^:private EntityRefOut+ "Entity ref as returned on reads. `:model` is any string: rows can predate a model string's retirement+ (serdes tolerates those too), and one legacy row must not fail response validation for a whole list."+ [:map+ [:model :string]+ [:id :int]+ [:name {:optional true} :string]])++(def ^:private Entry+ [:map+ [:id ms/PositiveInt]+ [:search_prompt :string]+ [:usage_instructions {:optional true} [:maybe :string]]+ [:entity EntityRefOut]+ [:verified :boolean]])++(def ^:private default-limit 50)+(def ^:private default-offset 0)++(api.macros/defendpoint :get "/"+ :- [:map+ [:data [:sequential Entry]]+ [:total :int]+ [:limit :int]+ [:offset :int]]+ "Get all curated search entries, paginated."+ [_route-params+ _query-params]+ (api/check-superuser)+ (let [limit (or (request/limit) default-limit)+ offset (or (request/offset) default-offset)]+ {:data (t2/select :model/CuratedSearchEntry+ {:order-by [[:id :asc]]+ :limit limit+ :offset offset})+ :total (t2/count :model/CuratedSearchEntry)+ :limit limit+ :offset offset}))++(api.macros/defendpoint :get "/:id"+ :- Entry+ "Get a curated search entry by ID."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ _query-params]+ (api/check-superuser)+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id)))++(api.macros/defendpoint :post "/"+ :- Entry+ "Create a new curated search entry."+ [_route-params+ _query-params+ {:keys [search_prompt usage_instructions entity verified]}+ :- [:map+ [:search_prompt ms/NonBlankString]+ [:usage_instructions {:optional true} [:maybe :string]]+ [:entity EntityRef]+ [:verified {:optional true} [:maybe :boolean]]]]+ (api/check-superuser)+ (t2/insert-returning-instance! :model/CuratedSearchEntry+ {:search_prompt search_prompt+ :usage_instructions usage_instructions+ :entity entity+ :verified (boolean verified)}))++(api.macros/defendpoint :put "/:id"+ :- Entry+ "Update a curated search entry by ID. Only the provided fields are changed; sending a null+ usage_instructions clears it."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ _query-params+ ;; patch semantics: presence of a key (not nil-ness of its value) decides what changes, so an explicit+ ;; null can clear the nullable usage_instructions. The non-nullable fields reject null in the schema.+ {:keys [search_prompt usage_instructions entity verified] :as body}+ :- [:map+ [:search_prompt {:optional true} ms/NonBlankString]+ [:usage_instructions {:optional true} [:maybe :string]]+ [:entity {:optional true} EntityRef]+ [:verified {:optional true} :boolean]]]+ (api/check-superuser)+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id))+ (let [changes (cond-> {}+ (contains? body :search_prompt) (assoc :search_prompt search_prompt)+ (contains? body :usage_instructions) (assoc :usage_instructions usage_instructions)+ (contains? body :entity) (assoc :entity entity)+ (contains? body :verified) (assoc :verified verified))]+ (when (seq changes)+ (t2/update! :model/CuratedSearchEntry id changes)))+ (t2/select-one :model/CuratedSearchEntry :id id))++#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}+(api.macros/defendpoint :delete "/:id"+ "Delete a curated search entry by ID."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]+ _query-params]+ (api/check-superuser)+ (api/check-404 (t2/select-one :model/CuratedSearchEntry :id id))+ (t2/delete! :model/CuratedSearchEntry :id id)+ api/generic-204-no-content)diff --git a/src/metabase/curated_search/core.clj b/src/metabase/curated_search/core.cljnew file mode 100644index 00000000000..d02732f02d6--- /dev/null+++ b/src/metabase/curated_search/core.clj@@ -0,0 +1,16 @@+(ns metabase.curated-search.core+ "Public API of the `curated-search` module.++ The curated search index is a hand-curated library of saved search prompts, each mapped to the entity that+ answers it (see [[metabase.curated-search.models.curated-search-entry]]).+ [[search]] matches a natural-language request against it by vector similarity (enterprise; returns []+ in OSS)."+ (:require+ [metabase.curated-search.mirror]+ [potemkin :as p]))++(comment metabase.curated-search.mirror/keep-me)++(p/import-vars+ [metabase.curated-search.mirror+ search])diff --git a/src/metabase/curated_search/mirror.clj b/src/metabase/curated_search/mirror.cljnew file mode 100644index 00000000000..5bca05b6b0d--- /dev/null+++ b/src/metabase/curated_search/mirror.clj@@ -0,0 +1,24 @@+(ns metabase.curated-search.mirror+ "OSS entry points for the enterprise pgvector mirror of the `curated_search_entries` table.++ The appdb table is authoritative; the mirror carries one embedding per row and serves the+ `search_curated` Metabot tool's similarity search.+ These `defenterprise` shims route to [[metabase-enterprise.curated-search.core]] when an+ enterprise license with semantic search is present; otherwise they no-op or return empty."+ (:require+ [metabase.premium-features.core :refer [defenterprise]]))++(defenterprise request-sync!+ "Ask the background sync to reconcile the pgvector mirror with the appdb table soon; no-op in OSS.+ Fire-and-forget: never throws, does no embedding or pgvector work on the calling thread."+ metabase-enterprise.curated-search.core+ []+ nil)++(defenterprise search+ "Similarity-search the curated search index for the saved prompts nearest `user-search-prompt`.+ Returns up to `limit` matches shaped `{:saved_search_prompt :usage_instructions :entity :score}`,+ best blended score first; [] in OSS or when no pgvector store is configured."+ metabase-enterprise.curated-search.core+ [_user-search-prompt _limit]+ [])diff --git a/src/metabase/curated_search/models/curated_search_entry.clj b/src/metabase/curated_search/models/curated_search_entry.cljnew file mode 100644index 00000000000..d043176df17--- /dev/null+++ b/src/metabase/curated_search/models/curated_search_entry.clj@@ -0,0 +1,102 @@+(ns metabase.curated-search.models.curated-search-entry+ "The `curated_search_entries` appdb table: curated saved search prompts, each mapped to the single entity+ that answers it, plus agent-facing `usage_instructions`.++ This table is authoritative.+ An enterprise pgvector mirror carries one embedding per row and serves the `search_curated`+ Metabot tool's similarity search.+ Writes here only nudge the mirror's background sync ([[mirror/request-sync!]]); they never touch+ the embedding service or the pgvector store themselves."+ (:require+ [metabase.curated-search.mirror :as mirror]+ [metabase.models.interface :as mi]+ [metabase.models.serialization :as serdes]+ [metabase.util :as u]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/CuratedSearchEntry [_model] :curated_search_entries)++(doto :model/CuratedSearchEntry+ (derive :metabase/model)+ (derive :hook/entity-id)+ (derive :hook/timestamped?))++(t2/deftransforms :model/CuratedSearchEntry+ {:entity mi/transform-json})++;;; Each write nudges the enterprise background sync, which reconciles the pgvector mirror against this+;;; table. The nudge is fire-and-forget (no-op in OSS, error-swallowing in EE), so appdb writes never+;;; fail or slow down because of the mirror.++(t2/define-after-insert :model/CuratedSearchEntry+ [row]+ (mirror/request-sync!)+ row)++(t2/define-after-update :model/CuratedSearchEntry+ [row]+ (mirror/request-sync!)+ row)++(t2/define-before-delete :model/CuratedSearchEntry+ [row]+ (mirror/request-sync!)+ row)++;;; ------------------------------------------------- Serialization -------------------------------------------------++;;; `entity` is a polymorphic reference — `{:model "table"|"card"|"model"|"metric"|"question" :id <local-id>}`.+;;; On export we swap the local id for a portable reference (a Card's entity_id, or a Table's+;;; `[db schema table]` path) and reverse it on import, mirroring `serdes/export-viz-link-card`.++(def ^:private entity-model->toucan+ "Toucan model for each entity ref `:model` string. Tables get the table-fk treatment instead.+ All the card flavors are Cards (the `:model` string mirrors the `read_resource` resource type the+ agent uses to fetch the entity, not the underlying table)."+ {"card" :model/Card+ "model" :model/Card+ "metric" :model/Card+ "question" :model/Card})++(defn- export-entity-ref [{:keys [model id] :as entity}]+ (cond+ (= model "table") (assoc entity :id (serdes/*export-table-fk* id))+ (entity-model->toucan model) (assoc entity :id (serdes/*export-fk* id (entity-model->toucan model)))+ ;; Unmapped model string: leave the ref as-is rather than aborting the whole export+ ;; (same forgiving behavior as serdes/export-viz-link-card).+ :else entity))++(defn- import-entity-ref [{:keys [model id] :as entity}]+ (cond+ (= model "table") (assoc entity :id (serdes/*import-table-fk* id))+ (entity-model->toucan model) (assoc entity :id (serdes/*import-fk* id (entity-model->toucan model)))+ :else entity))++(defmethod serdes/make-spec "CuratedSearchEntry" [_model-name _opts]+ {:copy [:entity_id :search_prompt :usage_instructions :verified]+ :transform {:created_at (serdes/date)+ :updated_at (serdes/date)+ :entity {:export export-entity-ref+ :import import-entity-ref}}+ :defaults {:verified false}})++(defmethod serdes/hash-fields :model/CuratedSearchEntry+ [_model]+ [:search_prompt])++(defmethod serdes/generate-path "CuratedSearchEntry" [_ entity]+ (serdes/maybe-labeled "CuratedSearchEntry" entity :search_prompt))++(defmethod serdes/storage-path "CuratedSearchEntry" [entity _ctx]+ [{:label "curated_search_entries"}+ {:label (u/slugify (:search_prompt entity) {:unicode? true}) :key (:entity_id entity)}])++(defmethod serdes/dependencies "CuratedSearchEntry"+ [{{:keys [model id]} :entity}]+ ;; A referenced Table is synthesized on import if missing, so (like link cards) we depend on its Database,+ ;; not the Table itself. Card refs depend on the Card directly.+ (cond+ (= model "table") #{[{:model "Database" :id (first id)}]}+ (entity-model->toucan model) #{[{:model "Card" :id id}]}+ :else #{}))diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.cljindex 486c960317d..e6c9b91b7bd 100644--- a/src/metabase/metabot/agent/profiles.clj+++ b/src/metabase/metabot/agent/profiles.clj@@ -170,7 +170,10 @@:prompt-template "natural-language-querying-only.selmer":max-iterations 10:temperature 0.3+ ;; the nlq profile gets both the general instance search and, for users with the semantic-search+ ;; feature, the curated search tool (gated via its :feature-semantic-search capability).:tools [#'tools/nlq-search-tool+ #'tools/curated-search-tool#'tools/read-resource-tool#'tools/construct-notebook-query-tool#'tools/navigate-user-tooldiff --git a/src/metabase/metabot/capabilities.clj b/src/metabase/metabot/capabilities.cljindex 95a3f75b7da..5e9477ab84c 100644--- a/src/metabase/metabot/capabilities.clj+++ b/src/metabase/metabot/capabilities.clj@@ -39,7 +39,9 @@(premium-features/has-feature? :transforms-basic)):feature-transforms)(when (premium-features/has-feature? :transforms-python)- :feature-transforms-python)]))+ :feature-transforms-python)+ (when (premium-features/has-feature? :semantic-search)+ :feature-semantic-search)]))(defn capability-set"Build the full set of capability keywords for a request: the backend-inferred feature capabilitiesdiff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.cljindex f4707e9b0c2..f620aefd450 100644--- a/src/metabase/metabot/tools.clj+++ b/src/metabase/metabot/tools.clj@@ -15,6 +15,7 @@[metabase.metabot.tools.construct :as tools.construct][metabase.metabot.tools.create-alert :as tools.create-alert][metabase.metabot.tools.create-dashboard-subscription :as tools.create-dashboard-subscription]+ [metabase.metabot.tools.curated-search :as tools.curated-search][metabase.metabot.tools.document :as tools.document][metabase.metabot.tools.metadata :as tools.metadata][metabase.metabot.tools.navigation :as tools.navigation]@@ -41,6 +42,8 @@sql-search-toolnlq-search-tooltransform-search-tool]+ [tools.curated-search+ curated-search-tool][tools.constructconstruct-notebook-query-tool][tools.documentdiff --git a/src/metabase/metabot/tools/curated_search.clj b/src/metabase/metabot/tools/curated_search.cljnew file mode 100644index 00000000000..9520b8b348e--- /dev/null+++ b/src/metabase/metabot/tools/curated_search.clj@@ -0,0 +1,180 @@+(ns metabase.metabot.tools.curated-search+ "Metabot `search_curated` tool, backed by the curated search index.++ Instead of ranking the whole instance, it matches the user's request by vector similarity against a+ hand-curated library of saved search prompts, each mapped to the single entity that answers it.+ Each match also carries `usage_instructions`: curator guidance on how to use that entity.++ The index has several prompts per entity, so raw vector hits are deduped to distinct entities and a+ small number returned.+ The matched entity refs are hydrated into the same enriched search-result shape the general `search`+ tool returns (`portable_entity_id`, fully-qualified names, database names, metric base tables), so the+ agent can build a query inline without an extra `read_resource` round-trip.++ Even where a general search is also offered, the tool surfaces the raw cosine `similarity` of each+ match and flags low-confidence results, so a curated miss reads as a miss rather than a+ confident-but-wrong curated hit.+ The similarity search runs in the enterprise pgvector store via [[metabase.curated-search.core]]."+ (:require+ [clojure.string :as str]+ [metabase.curated-search.core :as curated-search]+ [metabase.metabot.scope :as scope]+ [metabase.metabot.tools.search :as tools.search]+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++(def ^:private default-limit+ "Distinct curated matches returned by default. The index has ~3 prompts per entity and embeddings cluster+ tightly by domain; 5 was too aggressive (benchmark 15: ~12 expected entities sat at rank 6–10 and were+ cut off), so 10 to recover that recall while still deduping siblings."+ 10)+(def ^:private max-limit 20)++(def ^:private over-fetch-factor+ "Raw per-prompt hits to pull before deduping to distinct entities (several prompts map to one entity)."+ 4)+(def ^:private over-fetch-cap 60)++(def ^:private weak-similarity-threshold+ "Raw cosine similarity below which a match is flagged low-confidence. Calibrated on this index:+ in-domain top-1 similarities run ~0.52–0.81, clearly out-of-domain ~0.15–0.22."+ 0.45)++(def ^:private user-search-prompt-desc+ (str "A natural-language description of the data the user wants. Matched by vector similarity "+ "against curated saved prompts; phrase it like the saved prompt you'd expect to find, "+ "not as keywords."))++(def ^:private limit-desc+ (str "Maximum number of distinct curated matches (default " default-limit ", max " max-limit ")."))++(def ^:private curated-search-schema+ [:map {:closed true}+ [:user_search_prompt [:string {:description user-search-prompt-desc}]]+ [:limit {:optional true} [:maybe [:int {:min 1 :max max-limit :description limit-desc}]]]])++(def ^:private instructions+ ;; refers to the match/note elements by name only — literal angle-bracket tags here would make the+ ;; surrounding <instructions> element malformed+ (str "Each match element is a curated entry: a saved prompt, its usage_instructions, and the entity to "+ "use (already hydrated with the ids, names and portable references you need). similarity is the "+ "raw cosine match strength; a match flagged confidence=\"weak\" (or a leading note element) means "+ "nothing in the curated layer clearly matches — don't build on it blindly; prefer asking the user "+ "to clarify or narrow the request, or use the general search tool for uncurated results."))++(defn- similarity+ "Raw cosine similarity (1 - distance) for a match, from its score breakdown."+ ^double [score]+ (or (some (fn [s] (when (= :similarity (:name s)) (double (:score s)))) (:scores score)) 0.0))++(defn- dedupe-by-entity+ "Collapse raw per-prompt results (sorted best-first) to distinct entities, keeping the best-scoring+ prompt for each."+ [results]+ (->> results+ (reduce (fn [[seen acc] r]+ ;; normalize the model so "card" and "question" (aliases for the same Card) dedupe+ ;; together — both hydrate to one record, so distinct keys would return it twice.+ (let [{:keys [model id]} (:entity r)+ k [(tools.search/ref-model->entity-type model) id]]+ (if (seen k) [seen acc] [(conj seen k) (conj acc r)])))+ [#{} []])+ second))++(defn- build-matches+ "Fetch, dedupe to distinct entities, hydrate each match's entity ref into a full search record, and+ return the top `n` the current user can read.+ Each match: `{:saved_search_prompt :usage_instructions :score :similarity :weak? :entity hydrated-hit}`."+ [user-search-prompt n]+ (let [raw (curated-search/search+ user-search-prompt (min over-fetch-cap (* over-fetch-factor n)))+ deduped (dedupe-by-entity raw)+ ;; Hydration permission-filters (entity-refs->search-results drops entities the user can't read),+ ;; so hydrate the whole deduped candidate set and take `n` from the readable survivors — taking+ ;; `n` first would let unreadable top hits shrink the result below `n` while readable matches sit+ ;; just past the cut.+ by-key (into {} (map (juxt (juxt :type :id) identity))+ (tools.search/entity-refs->search-results (distinct (map :entity deduped))))]+ (->> (for [{:keys [saved_search_prompt usage_instructions entity score]} deduped+ ;; hydrated records use the agent-facing entity type, so normalize the ref's model to match+ ;; (plain "card" refs hydrate as "question")+ :let [resolved (get by-key [(tools.search/ref-model->entity-type (:model entity)) (:id entity)])+ sim (similarity score)]+ :when resolved]+ {:saved_search_prompt saved_search_prompt+ :usage_instructions usage_instructions+ :score score+ :similarity sim+ :weak? (< sim weak-similarity-threshold)+ :entity resolved})+ (take n))))++(defn- match->xml [{:keys [saved_search_prompt usage_instructions score similarity weak? entity]}]+ ;; String/format with Locale/ROOT so %.3f uses a '.' decimal separator regardless of the instance's+ ;; site locale (clojure.core/format always uses the default locale).+ (str (String/format java.util.Locale/ROOT "<match score=\"%.3f\" similarity=\"%.3f\" confidence=\"%s\">\n"+ (object-array [(double (:total_score score)) (double similarity)+ (if weak? "weak" "strong")]))+ "<saved_search_prompt>" (llm-shape/escape-xml saved_search_prompt) "</saved_search_prompt>\n"+ (when-not (str/blank? usage_instructions)+ (str "<usage_instructions>" (llm-shape/escape-xml usage_instructions) "</usage_instructions>\n"))+ (llm-shape/search-result->xml entity)+ "\n</match>"))++(defn- format-output [matches]+ (if (empty? matches)+ "<search_results>No matching curated entities.</search_results>"+ (str (when (:weak? (first matches))+ (str "<note>No strong curated match for this request — treat the results below as weak "+ "guesses; consider asking the user to clarify or narrow the request.</note>\n"))+ "<search_results>\n"+ (str/join "\n" (map match->xml matches))+ "\n</search_results>\n"+ "<instructions>" instructions "</instructions>")))++(defn- flatten-data+ "Flatten matches to a per-entity list (the `:result-type :search` shape), each entity record annotated+ with its match's prompt, usage_instructions, similarity and confidence."+ [matches]+ (mapv (fn [{:keys [saved_search_prompt usage_instructions score similarity weak? entity]}]+ (assoc entity+ :saved_search_prompt saved_search_prompt+ :usage_instructions usage_instructions+ :score score+ :similarity similarity+ :confidence (if weak? "weak" "strong")))+ matches))++(mu/defn ^{:tool-name "search_curated"+ :scope scope/agent-search+ ;; only offered to users whose license includes semantic search (the mirror's backing store);+ ;; :feature-semantic-search is derived in profiles/feature-capabilities.+ :capabilities #{:feature-semantic-search}}+ curated-search-tool+ "Find the best data to answer the user's request from the curated search index — a vetted library of+ saved prompts, each mapped to the entity that answers it. Phrase `user_search_prompt` as a full+ natural-language description of the data wanted (it is matched on meaning, not keywords).++ Prefer this over the general `search` tool: try it first, and fall back to `search` only when no+ curated match is strong, or to complement a curated match with broader (uncurated) results.++ Returns a handful of distinct curated matches, best-first. Each match has `saved_search_prompt`, curator+ `usage_instructions`, a raw `similarity`, and the `entity` — a full search record (name, type, database,+ `portable_entity_id`, fully-qualified name) you can use directly. If the top match is flagged+ low-confidence (a leading <note> / confidence=\"weak\"), nothing in the curated layer clearly matches —+ prefer asking the user to clarify, or fall back to the general `search` tool for uncurated results."+ [{:keys [user_search_prompt limit]} :- curated-search-schema]+ (try+ (let [n (min max-limit (or limit default-limit))+ matches (build-matches user_search_prompt n)]+ {:output (format-output matches)+ :structured-output {:result-type :search+ :data (flatten-data matches)+ :total_count (count matches)+ :weak_match (boolean (:weak? (first matches)))}})+ (catch Exception e+ (log/error e "Error in curated search")+ {:output (str "search_curated failed: " (or (ex-message e) "Unknown error"))})))diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 716a6d28716..51feec73c2d 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -11,6 +11,7 @@[metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]+ [metabase.models.interface :as mi][metabase.permissions.core :as perms][metabase.search.core :as search][metabase.search.engine :as search.engine]@@ -307,6 +308,81 @@enrich-with-metric-base-tablesremove-unreadable-transforms)))+(defn- table-refs->results+ [ids]+ (when (seq ids)+ ;; only surface tables the current user can read — a curated entry may point at one they can't access+ (for [t (filter mi/can-read?+ (t2/select [:model/Table :id :name :display_name :db_id :schema :description] :id [:in ids]))]+ {:id (:id t)+ :type "table"+ :name (:name t)+ :display_name (:display_name t)+ :database_id (:db_id t)+ :database_schema (:schema t)+ :description (:description t)})))++(defn- card-refs->results+ "Build post-processed search-result records for card-backed refs (`{:id .. :type \"model\"|\"metric\"|\"question\"}`).+ Emits one record per ref, so the same card registered under two type strings yields a record for each+ (rather than collapsing to one and silently dropping the other)."+ [refs]+ (let [ids (distinct (map :id refs))+ ;; only surface cards the current user can read (collection perms) — see table-refs->results+ id->card (when (seq ids)+ (into {} (map (juxt :id identity))+ (filter mi/can-read?+ (t2/select [:model/Card :id :name :description :database_id :collection_id :card_schema]+ :id [:in ids]))))+ coll-ids (->> (vals id->card) (keep :collection_id) distinct)+ id->coll (when (seq coll-ids)+ (into {} (map (juxt :id identity))+ (t2/select [:model/Collection :id :name :authority_level] :id [:in coll-ids])))+ ;; verified is already a set (t2/select-fn-set), possibly nil when there were no ids+ verified (when (seq ids)+ (t2/select-fn-set :moderated_item_id :model/ModerationReview+ :moderated_item_id [:in ids] :moderated_item_type "card"+ :most_recent true :status "verified"))]+ (for [{:keys [id type]} refs+ :let [c (id->card id)]+ :when c]+ (let [coll (get id->coll (:collection_id c))]+ {:id id+ :type type+ :name (:name c)+ :description (:description c)+ :database_id (:database_id c)+ :verified (contains? verified id)+ :collection (when coll (select-keys coll [:id :name :authority_level]))}))))++(defn ref-model->entity-type+ "Normalize an entity ref's `:model` string to the agent-facing entity type: plain cards are+ `\"question\"` everywhere the agent sees them (`read_resource` URIs, search results)."+ [model]+ (if (= model "card") "question" model))++(defn entity-refs->search-results+ "Hydrate semantic-layer entity refs into the enriched search-result shape that+ [[metabase.metabot.tools.shared.llm-shape/search-results->xml]] and the `search` tool consume.++ `refs` is a seq of `{:model <entity-type> :id <id>}` where `<entity-type>` is `\"table\"`, `\"model\"`,+ `\"metric\"`, or `\"question\"` (the names the agent uses with `read_resource`); `\"card\"` is accepted+ and normalized to `\"question\"`.+ Returns records carrying `:portable_entity_id`, `:database_name`, fully-qualified names, metric base+ tables, etc. — everything the LLM needs to build a query without an extra round-trip.+ Refs whose entity no longer exists are dropped."+ [refs]+ (let [by-model (group-by (comp ref-model->entity-type :model) refs)+ table-ids (distinct (map :id (get by-model "table")))+ card-refs (for [m ["model" "metric" "question"], r (get by-model m)] {:id (:id r) :type m})]+ (->> (concat (table-refs->results table-ids)+ (card-refs->results (distinct card-refs)))+ enrich-with-collection-descriptions+ enrich-with-database-engines+ enrich-with-portable-entity-ids+ enrich-with-metric-base-tables+ remove-unreadable-transforms)))+(defn- format-search-output"Format search results as an LLM-ready string."[results]diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.cljindex 35733ef1a12..dce8ed5bd80 100644--- a/src/metabase/metabot/tools/shared/llm_shape.clj+++ b/src/metabase/metabot/tools/shared/llm_shape.clj@@ -73,9 +73,10 @@(seq data))(str "```json\n" (json/encode data {:pretty true}) "\n```")))-(defn- escape-xml+(defn escape-xml"Escape XML special characters in a string.- Only needed for content that bypasses Selmer's auto-escaping (marked with |safe)."+ Only needed for content that bypasses Selmer's auto-escaping (marked with |safe) or is interpolated+ into hand-built XML."[s](when s(-> (str s)diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 0ef42e1e353..35498a35801 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -33,6 +33,7 @@:model/CommentReaction metabase.comments.models.comment-reaction:model/ConnectionImpersonation metabase-enterprise.impersonation.models:model/ContentTranslation metabase.content-translation.models+ :model/CuratedSearchEntry metabase.curated-search.models.curated-search-entry:model/CustomVizPlugin metabase-enterprise.custom-viz-plugin.models.custom-viz-plugin:model/Dashboard metabase.dashboards.models.dashboard:model/DashboardBookmark metabase.bookmarks.models.bookmarkdiff --git a/test/metabase/curated_search/api_test.clj b/test/metabase/curated_search/api_test.cljnew file mode 100644index 00000000000..2b4ad211fcf--- /dev/null+++ b/test/metabase/curated_search/api_test.clj@@ -0,0 +1,129 @@+(ns metabase.curated-search.api-test+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defmacro ^:private with-test-entry+ "Create a curated search entry for testing and bind it to `sym`, deleting it on exit."+ [[sym attrs] & body]+ `(let [~sym (t2/insert-returning-instance! :model/CuratedSearchEntry+ (merge {:search_prompt "find orders"+ :entity {:model "table" :id 1}+ :verified false}+ ~attrs))]+ (try+ ~@body+ (finally+ (t2/delete! :model/CuratedSearchEntry :id (:id ~sym))))))++(deftest list-test+ (with-test-entry [entry {}]+ (testing "superuser can list curated search entries"+ (let [response (mt/user-http-request :crowberto :get 200 "curated-search/")]+ (is (contains? response :data))+ (is (contains? response :total))+ (is (some #(= (:id entry) (:id %)) (:data response)))))+ (testing "non-superuser gets 403"+ (mt/user-http-request :rasta :get 403 "curated-search/"))+ (testing "a legacy row whose entity model is no longer in the write enum still reads back fine"+ (with-test-entry [legacy {:entity {:model "retired-model-string" :id 7}}]+ (is (some #(= (:id legacy) (:id %))+ (:data (mt/user-http-request :crowberto :get 200 "curated-search/"))))))))++(deftest get-test+ (with-test-entry [entry {:search_prompt "find customers" :verified true}]+ (testing "superuser can fetch a curated search entry by id"+ (is (=? {:id (:id entry)+ :search_prompt "find customers"+ :entity {:model "table" :id 1}+ :verified true}+ (mt/user-http-request :crowberto :get 200 (str "curated-search/" (:id entry))))))+ (testing "returns 404 for unknown id"+ (mt/user-http-request :crowberto :get 404 "curated-search/0"))+ (testing "non-superuser gets 403"+ (mt/user-http-request :rasta :get 403 (str "curated-search/" (:id entry))))))++(deftest create-test+ (testing "superuser can create a curated search entry"+ (let [response (mt/user-http-request :crowberto :post 200 "curated-search/"+ {:search_prompt "find revenue"+ :usage_instructions "Use the Revenue card."+ :entity {:model "card" :id 42}})]+ (try+ (is (=? {:search_prompt "find revenue"+ :usage_instructions "Use the Revenue card."+ :entity {:model "card" :id 42}+ :verified false}+ response))+ (finally+ (t2/delete! :model/CuratedSearchEntry :id (:id response))))))+ (testing "verified defaults to false and usage_instructions to nil when omitted"+ (let [response (mt/user-http-request :crowberto :post 200 "curated-search/"+ {:search_prompt "find users"+ :entity {:model "table" :id 1}})]+ (try+ (is (false? (:verified response)))+ (is (nil? (:usage_instructions response)))+ (finally+ (t2/delete! :model/CuratedSearchEntry :id (:id response))))))+ (testing "entity is required"+ (mt/user-http-request :crowberto :post 400 "curated-search/"+ {:search_prompt "find users"}))+ (testing "an unknown entity model is rejected"+ (mt/user-http-request :crowberto :post 400 "curated-search/"+ {:search_prompt "find users"+ :entity {:model "garbage" :id 1}}))+ (testing "a blank search_prompt is rejected"+ (mt/user-http-request :crowberto :post 400 "curated-search/"+ {:search_prompt " " :entity {:model "table" :id 1}}))+ (testing "non-superuser gets 403"+ (mt/user-http-request :rasta :post 403 "curated-search/"+ {:search_prompt "find orders" :entity {:model "table" :id 1}})))++(deftest update-test+ (with-test-entry [entry {}]+ (testing "superuser can update the prompt"+ (let [updated (mt/user-http-request :crowberto :put 200+ (str "curated-search/" (:id entry))+ {:search_prompt "updated prompt"})]+ (is (= "updated prompt" (:search_prompt updated)))))+ (testing "superuser can update usage_instructions and the entity"+ (let [updated (mt/user-http-request :crowberto :put 200+ (str "curated-search/" (:id entry))+ {:usage_instructions "Join orders to customers on customer_id."+ :entity {:model "model" :id 7}})]+ (is (=? {:usage_instructions "Join orders to customers on customer_id."+ :entity {:model "model" :id 7}}+ updated))))+ (testing "an explicit null clears usage_instructions"+ (let [updated (mt/user-http-request :crowberto :put 200+ (str "curated-search/" (:id entry))+ {:usage_instructions nil})]+ (is (nil? (:usage_instructions updated)))))+ (testing "the non-nullable fields reject an explicit null"+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))+ {:entity nil})+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))+ {:verified nil}))+ (testing "a blank search_prompt is rejected"+ (mt/user-http-request :crowberto :put 400 (str "curated-search/" (:id entry))+ {:search_prompt " "}))+ (testing "returns 404 for unknown id"+ (mt/user-http-request :crowberto :put 404 "curated-search/0" {:search_prompt "x"}))+ (testing "non-superuser gets 403"+ (mt/user-http-request :rasta :put 403+ (str "curated-search/" (:id entry))+ {:search_prompt "x"}))))++(deftest delete-test+ (with-test-entry [entry {}]+ (testing "non-superuser gets 403"+ (mt/user-http-request :rasta :delete 403 (str "curated-search/" (:id entry))))+ (testing "superuser can delete an entry"+ (mt/user-http-request :crowberto :delete 204 (str "curated-search/" (:id entry)))+ (is (nil? (t2/select-one :model/CuratedSearchEntry :id (:id entry)))))+ (testing "returns 404 for unknown id"+ (mt/user-http-request :crowberto :delete 404 "curated-search/0"))))diff --git a/test/metabase/curated_search/models/curated_search_entry_test.clj b/test/metabase/curated_search/models/curated_search_entry_test.cljnew file mode 100644index 00000000000..671bc71e23f--- /dev/null+++ b/test/metabase/curated_search/models/curated_search_entry_test.clj@@ -0,0 +1,58 @@+(ns metabase.curated-search.models.curated-search-entry-test+ (:require+ [clojure.test :refer :all]+ [metabase.curated-search.mirror :as mirror]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private table-entity {:model "table" :id 42})++(deftest entity-json-roundtrips-test+ (testing "entity is a JSON object and timestamps are populated"+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}+ {:search_prompt "monthly revenue by region" :entity table-entity}]+ (is (=? {:search_prompt "monthly revenue by region"+ :entity table-entity+ :verified false+ :created_at some?+ :updated_at some?}+ (t2/select-one :model/CuratedSearchEntry :id id))))))++(deftest usage-instructions-persist-test+ (testing "usage_instructions persists alongside the prompt"+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}+ {:search_prompt "monthly revenue by region" :entity table-entity+ :usage_instructions "Use this metric for total revenue; group by month."}]+ (is (= "Use this metric for total revenue; group by month."+ (t2/select-one-fn :usage_instructions :model/CuratedSearchEntry :id id)))))+ (testing "usage_instructions is optional (nil when omitted)"+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}+ {:search_prompt "orders" :entity table-entity}]+ (is (nil? (t2/select-one-fn :usage_instructions :model/CuratedSearchEntry :id id))))))++(deftest hooks-nudge-the-background-sync-test+ (testing "insert, update and delete each request a background sync — and do nothing else"+ (let [nudges (atom 0)]+ (mt/with-dynamic-fn-redefs [mirror/request-sync! (fn [] (swap! nudges inc) nil)]+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}+ {:search_prompt "p1" :entity table-entity}]+ (is (= 1 @nudges) "insert nudges once")+ (t2/update! :model/CuratedSearchEntry id {:search_prompt "p1 updated"})+ (is (= 2 @nudges) "update nudges once")+ (t2/delete! :model/CuratedSearchEntry :id id)+ (is (= 3 @nudges) "delete nudges once"))))))++(deftest hooks-never-break-appdb-writes-test+ (testing "insert/update/delete succeed even though the mirror is unavailable in tests"+ ;; The hooks call the OSS defenterprise shim, which no-ops without an enterprise license. They must+ ;; never throw or fail the authoritative write.+ (mt/with-temp [:model/CuratedSearchEntry {:keys [id]}+ {:search_prompt "p1" :entity table-entity}]+ (testing "update"+ (is (pos? (t2/update! :model/CuratedSearchEntry id {:search_prompt "p1 updated"})))+ (is (= "p1 updated" (t2/select-one-fn :search_prompt :model/CuratedSearchEntry :id id))))+ (testing "delete"+ (is (pos? (t2/delete! :model/CuratedSearchEntry :id id)))+ (is (nil? (t2/select-one :model/CuratedSearchEntry :id id)))))))diff --git a/test/metabase/metabot/agent/profiles_test.clj b/test/metabase/metabot/agent/profiles_test.cljindex 9bc90cff472..af6fd477f3a 100644--- a/test/metabase/metabot/agent/profiles_test.clj+++ b/test/metabase/metabot/agent/profiles_test.clj@@ -64,7 +64,9 @@(is (= "anthropic/claude-sonnet-4-6" (:model profile)))(is (= 10 (:max-iterations profile)))(is (= 0.3 (:temperature profile)))+ ;; nlq offers both the general search and the curated search tool (the latter feature-gated)(is (contains? (tool-names profile) "search"))+ (is (contains? (tool-names profile) "search_curated"))(is (contains? (tool-names profile) "construct_notebook_query"))))(testing "retrieves slackbot profile"(let [profile (profiles/get-profile :slackbot)]@@ -156,11 +158,12 @@"SQL tools should be gated by permission:write_sql_queries"))))))(deftest embedding-next-matches-nlq-tools-test- (testing "embedding_next and nlq profiles have identical tool sets"+ (testing "nlq has the same tools as embedding_next plus the (feature-gated) curated search tool"(let [tool-names (fn [profile] (set (map #(:tool-name (meta %)) (:tools profile))))embedding (profiles/get-profile :embedding_next)nlq (profiles/get-profile :nlq)]- (is (= (tool-names nlq) (tool-names embedding)))))+ (is (= (disj (tool-names nlq) "search_curated")+ (tool-names embedding)))))(binding [scope/*current-user-scope* api-scope/unrestricted](testing "navigate_user is excluded without the capability"(let [tools (profiles/get-tools-for-profile :embedding_next [])]diff --git a/test/metabase/metabot/tools/curated_search_test.clj b/test/metabase/metabot/tools/curated_search_test.cljnew file mode 100644index 00000000000..46837448c69--- /dev/null+++ b/test/metabase/metabot/tools/curated_search_test.clj@@ -0,0 +1,146 @@+(ns metabase.metabot.tools.curated-search-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.curated-search.core :as cs.core]+ [metabase.metabot.tools.curated-search :as curated-search]+ [metabase.metabot.tools.search :as tools.search]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest tool-shape-oss-fallback-test+ (testing "without the semantic-search feature the tool returns the standard empty result shape"+ ;; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient+ ;; premium token (e.g. an all-features dev token) or rows left in a local pgvector store.+ (mt/with-premium-features #{}+ (is (=? {:output #(str/includes? % "No matching curated entities")+ :structured-output {:result-type :search+ :data []+ :total_count 0+ :weak_match false}}+ (curated-search/curated-search-tool+ {:user_search_prompt "revenue per region"}))))))++(deftest similarity-helper-test+ (let [similarity (var-get #'curated-search/similarity)]+ (testing "similarity is pulled from the :similarity score factor"+ (is (= 0.8 (similarity {:scores [{:name :similarity :score 0.8} {:name :verified :score 1.0}]})))+ (is (= 0.0 (similarity {:scores [{:name :verified :score 1.0}]}))))))++(deftest build-matches-permission-filters-before-take-test+ (testing "hydration permission-filters the whole candidate set before take-n, so an unreadable top hit doesn't crowd out readable matches just past the cut"+ (let [build-matches (var-get #'curated-search/build-matches)+ ;; three distinct entities, best-first+ raw (vec (for [id [1 2 3]]+ {:entity {:model "table" :id id}+ :saved_search_prompt (str "p" id)+ :usage_instructions ""+ :score {:total_score (- 1.0 (/ id 10.0))+ :scores [{:name :similarity :score 0.8}]}}))]+ (mt/with-dynamic-fn-redefs [cs.core/search (fn [_ _] raw)+ ;; entity 1 (the top hit) is unreadable → dropped during hydration+ tools.search/entity-refs->search-results+ (fn [refs] (for [{:keys [model id]} refs :when (not= 1 id)]+ {:type model :id id :name (str "e" id)}))]+ (testing "asking for 2 returns the 2 readable entities (2 and 3), not just the 1 below the old top-2 cut"+ (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2)))))))))++(deftest match->xml-locale-independent-test+ (testing "score/similarity render with a '.' decimal separator even under a comma-decimal default locale"+ (let [match->xml (var-get #'curated-search/match->xml)+ orig (java.util.Locale/getDefault)]+ (try+ (java.util.Locale/setDefault (java.util.Locale/forLanguageTag "de-DE"))+ (let [out (match->xml {:saved_search_prompt "p" :usage_instructions ""+ :score {:total_score 0.95} :similarity 0.78 :weak? false+ :entity {:type "table" :id 1 :name "T" :database_id 1}})]+ (is (str/includes? out "score=\"0.950\""))+ (is (str/includes? out "similarity=\"0.780\"")))+ (finally+ (java.util.Locale/setDefault orig))))))++(deftest dedupe-by-entity-test+ (let [dedupe (var-get #'curated-search/dedupe-by-entity)+ results [{:entity {:model "metric" :id 9} :score {:total_score 0.95}} ; entity 9, best+ {:entity {:model "metric" :id 9} :score {:total_score 0.80}} ; entity 9 again (sibling prompt)+ {:entity {:model "table" :id 1} :score {:total_score 0.70}}]]+ (testing "collapses to distinct entities, keeping the first (best-scoring) occurrence"+ (let [out (dedupe results)]+ (is (= 2 (count out)))+ (is (= [0.95 0.70] (mapv (comp :total_score :score) out)))))+ (testing "\"card\" and \"question\" are aliases for the same Card, so they collapse to one entity"+ (let [out (dedupe [{:entity {:model "card" :id 5} :score {:total_score 0.9}}+ {:entity {:model "question" :id 5} :score {:total_score 0.8}}])]+ (is (= 1 (count out)))+ (is (= 0.9 (-> out first :score :total_score)))))))++(deftest format-output-test+ (let [format-output (var-get #'curated-search/format-output)+ ;; matches as produced by build-matches: :entity is already a full search-result record.+ matches [{:saved_search_prompt "monthly revenue by region"+ :usage_instructions "Use the Revenue metric; group by month."+ :similarity 0.78 :weak? false+ :score {:total_score 0.95}+ :entity {:type "metric" :id 9 :name "Revenue" :database_id 1+ :portable_entity_id "abcDEF"}}+ {:saved_search_prompt "orders joined to customers"+ :usage_instructions ""+ :similarity 0.61 :weak? false+ :score {:total_score 0.80}+ :entity {:type "table" :id 1 :name "ORDERS" :database_schema "PUBLIC"+ :database_id 1}}]+ out (format-output matches)]+ (testing "renders score, similarity, confidence, the saved prompt, and hydrated entity details"+ (is (str/includes? out "score=\"0.950\""))+ (is (str/includes? out "similarity=\"0.780\""))+ (is (str/includes? out "confidence=\"strong\""))+ (is (str/includes? out "<saved_search_prompt>monthly revenue by region</saved_search_prompt>"))+ (is (str/includes? out "Revenue"))+ (is (str/includes? out "ORDERS")))+ (testing "usage instructions surface when present and are omitted when blank"+ (is (str/includes? out "<usage_instructions>Use the Revenue metric; group by month.</usage_instructions>"))+ (is (= 1 (count (re-seq #"<usage_instructions>" out)))))+ (testing "no leading weak <note> when the top match is strong"+ (is (not (str/starts-with? out "<note>"))))))++(deftest xml-escaping-test+ (let [format-output (var-get #'curated-search/format-output)+ matches [{:saved_search_prompt "P&L by quarter <2026>"+ :usage_instructions "Profit & loss; values < 0 are losses."+ :similarity 0.7 :weak? false+ :score {:total_score 0.7}+ :entity {:type "table" :id 1 :name "FINANCE" :database_id 1}}]+ out (format-output matches)]+ (testing "curator-entered text is escaped in the XML output"+ (is (str/includes? out "<saved_search_prompt>P&L by quarter <2026></saved_search_prompt>"))+ (is (str/includes? out "<usage_instructions>Profit & loss; values < 0 are losses.</usage_instructions>")))))++(deftest weak-match-note-test+ (let [format-output (var-get #'curated-search/format-output)+ matches [{:saved_search_prompt "something only loosely related"+ :usage_instructions "" :similarity 0.20 :weak? true+ :score {:total_score 0.20}+ :entity {:type "table" :id 7 :name "MISC" :database_id 1}}]+ out (format-output matches)]+ (testing "a weak top match prepends the cautionary note and marks confidence weak"+ (is (str/starts-with? out "<note>"))+ (is (str/includes? out "No strong curated match"))+ (is (str/includes? out "confidence=\"weak\"")))))++(deftest flatten-data-test+ (let [flatten-data (var-get #'curated-search/flatten-data)+ matches [{:saved_search_prompt "p1" :usage_instructions "u1"+ :score {:total_score 0.9} :similarity 0.7 :weak? false+ :entity {:type "metric" :id 9 :name "Revenue"}}+ {:saved_search_prompt "p2" :usage_instructions "u2"+ :score {:total_score 0.6} :similarity 0.3 :weak? true+ :entity {:type "table" :id 1 :name "ORDERS"}}]+ out (flatten-data matches)]+ (testing "one annotated record per match"+ (is (= 2 (count out)))+ (is (=? [{:type "metric" :id 9 :saved_search_prompt "p1" :usage_instructions "u1"+ :confidence "strong" :similarity 0.7}+ {:type "table" :id 1 :saved_search_prompt "p2" :usage_instructions "u2"+ :confidence "weak" :similarity 0.3}]+ out)))))diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex 70ff10d75a8..edefca12e38 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -422,6 +422,66 @@(is (some? dash-res) "expected the dashboard to appear in search results")(is (not (contains? dash-res :portable_entity_id))))))))))+(deftest entity-refs->search-results-test+ (testing "hydrates {:model :id} refs (as stored by the semantic layer) into enriched search records"+ (mt/with-test-user :crowberto+ (mt/with-temp [:model/Card {m-id :id m-eid :entity_id}+ {:name "Hydrate Sample Model" :type :model+ :database_id (mt/id) :table_id (mt/id :orders)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (mt/id :orders)}}}]+ (mt/with-temp [:model/Card {q-id :id} {:name "Hydrate Sample Question"+ :database_id (mt/id) :table_id (mt/id :orders)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (mt/id :orders)}}}]+ (let [results (search/entity-refs->search-results+ [{:model "model" :id m-id}+ {:model "table" :id (mt/id :orders)}+ {:model "card" :id q-id} ; normalized to "question"+ {:model "model" :id Integer/MAX_VALUE}]) ; nonexistent → dropped+ by-id (into {} (map (juxt (juxt :type :id) identity)) results)]+ (testing "model ref hydrates with type, name, and portable_entity_id"+ (is (=? {:type "model" :name "Hydrate Sample Model" :portable_entity_id m-eid+ :database_id (mt/id)}+ (get by-id ["model" m-id]))))+ (testing "table ref hydrates with type table and a database name"+ (is (=? {:type "table" :database_id (mt/id) :database_name string?}+ (get by-id ["table" (mt/id :orders)]))))+ (testing "a card ref hydrates as the agent-facing type question"+ (is (=? {:type "question" :name "Hydrate Sample Question"}+ (get by-id ["question" q-id]))))+ (testing "refs whose entity no longer exists are dropped"+ (is (= 3 (count results))))))))))++(deftest entity-refs->search-results-same-card-two-types-test+ (testing "a card referenced under two type strings hydrates to one record per ref (neither is dropped)"+ (mt/with-test-user :crowberto+ (mt/with-temp [:model/Card {c-id :id} {:name "Dual Typed"+ :database_id (mt/id) :table_id (mt/id :orders)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (mt/id :orders)}}}]+ (let [results (search/entity-refs->search-results+ [{:model "model" :id c-id} {:model "metric" :id c-id}])]+ (is (= #{["model" c-id] ["metric" c-id]}+ (set (map (juxt :type :id) results)))))))))++(deftest entity-refs->search-results-respects-read-perms-test+ (testing "hydration drops entities the current user can't read — a curated entry may point at a restricted one"+ (mt/with-non-admin-groups-no-root-collection-perms+ (mt/with-temp [:model/Collection {coll-id :id} {}+ :model/Card {restricted-id :id}+ {:name "Secret Card" :collection_id coll-id+ :database_id (mt/id) :table_id (mt/id :orders)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (mt/id :orders)}}}]+ (let [refs [{:model "card" :id restricted-id}]]+ (testing "a superuser can read it"+ (mt/with-test-user :crowberto+ (is (= [restricted-id] (map :id (search/entity-refs->search-results refs))))))+ (testing "a user without access to its collection does not see it"+ (mt/with-test-user :rasta+ (is (empty? (search/entity-refs->search-results refs))))))))))+(deftest enrich-with-metric-base-tables-test(testing (str "Metric search results carry `base_table_*` fields so the LLM can write\n""`source-table:` without a separate entity_details call. We look up\n"diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.cljindex 4634ef98c9a..d77b8a4e3da 100644--- a/test/metabase/metabot/tools_test.clj+++ b/test/metabase/metabot/tools_test.clj@@ -67,12 +67,17 @@(is (contains? tools "read_resource"))(is (contains? tools "ask_for_sql_clarification"))))-(deftest ^:parallel get-tools-for-nlq-profile-test- (let [tools (tools-for-profile :nlq)]- (is (map? tools))- (is (contains? tools "search"))- (is (contains? tools "construct_notebook_query"))- (is (contains? tools "create_chart"))))+(deftest get-tools-for-nlq-profile-test+ (testing "the general search is always available; the curated search tool only with the semantic-search feature"+ (mt/with-premium-features #{}+ (let [tools (tools-for-profile :nlq)]+ (is (map? tools))+ (is (contains? tools "search"))+ (is (contains? tools "construct_notebook_query"))+ (is (contains? tools "create_chart"))+ (is (not (contains? tools "search_curated")))))+ (mt/with-premium-features #{:semantic-search}+ (is (contains? (tools-for-profile :nlq) "search_curated")))))(deftest ^:parallel get-tools-for-document-generate-content-profile-test(let [tools (tools-for-profile :document-generate-content)]