Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Support pgvector on the app DB with minimal config (#77371)

Chris Truter8ef62d8612b4
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index dc6faf705dc..f20490a8ec6 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -301,12 +301,79 @@ jobs:
previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}
run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
+ # pgvector-on-the-app-db mode: the app db itself is a pgvector-capable Postgres and
+ # MB_PGVECTOR_DB_URL is deliberately unset. The dedicated-harness tests gate themselves off; the
+ # appdb-pgvector-mode round trip (and the URL-independent unit tests) run for real.
+ semantic-search-tests-appdb-mode:
+ if: ${{ !inputs.skip }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 40
+ strategy:
+ fail-fast: false
+ matrix:
+ version:
+ - name: pgvector pg17 as app db, no dedicated pgvector
+ junit-name: semantic-search-tests-appdb-mode-pg17
+ appdb-image: pgvector/pgvector:pg17
+ job:
+ - name: Semantic Search Tests
+ build-static-viz: false
+ test-args: >-
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/entity_retrieval" "enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj"]'
+ exclude-tag: ':mb/driver-tests'
+
+ services:
+ postgres:
+ image: ${{ matrix.version.appdb-image }}
+ ports:
+ - "5432:5432"
+ env:
+ POSTGRES_USER: mb_test
+ POSTGRES_DB: mb_test
+ POSTGRES_HOST_AUTH_METHOD: trust
+ env:
+ CI: 'true'
+ MB_DB_TYPE: postgres
+ MB_DB_PORT: 5432
+ MB_DB_HOST: localhost
+ MB_DB_DBNAME: mb_test
+ MB_DB_USER: mb_test
+ # opts this job into the destructive appdb-pgvector-mode round trip, which then FAILS (rather
+ # than silently skipping) if the app db can't actually host pgvector
+ MB_APPDB_PGVECTOR_MODE_TEST: 'true'
+ name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
+ steps:
+ - uses: actions/checkout@v6
+ - name: Test ${{ matrix.version.name }}
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ build-static-viz: ${{ matrix.job.build-static-viz }}
+ junit-name: ${{ matrix.version.junit-name }}
+ test-args: >-
+ ${{ matrix.job.test-args }}
+ :exclude-tags '[${{ matrix.job.exclude-tag }}]'
+ quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
+ - name: Upload Test Results
+ uses: ./.github/actions/upload-test-results
+ if: always()
+ with:
+ input-path: ./target/junit/
+ output-name: ${{ github.job }}
+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+ role-to-assume: ${{ secrets.CI_TEST_RESULTS_IAM_ROLE }}
+ aws-region: ${{ vars.AWS_REGION }}
+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+ previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}
+ run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
+
semantic-search-tests-result:
needs:
- semantic-search-tests-h2
- semantic-search-tests-mariadb
- semantic-search-tests-mysql
- semantic-search-tests-postgres
+ - semantic-search-tests-appdb-mode
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 5
name: semantic-search-tests-result
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
index 086872b2b5c..f18db30057a 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
@@ -30,11 +30,16 @@
(jobs/key "metabase-enterprise.entity-retrieval.sync.job"))
(defn pgvector-configured?
- "True when a pgvector store is configured (read once at boot from MB_PGVECTOR_DB_URL).
+ "True when a dedicated pgvector store is configured (MB_PGVECTOR_DB_URL).
+ Deliberately excludes pgvector-on-the-app-db: entity retrieval's tables are not yet schema-isolated
+ (bare `library_entity_index*` names, dropped and recreated on rebuild), so it stays dedicated-only
+ until they are.
The sync task gates scheduling on this rather than [[available?]], so the periodic safety net exists
- even when the library-retrieval feature is turned on after startup (a common onboarding flow)."
+ even when the library-retrieval feature is turned on after startup (a common onboarding flow).
+ A plain URL check, not [[metabase-enterprise.semantic-search.db.datasource/pgvector-mode]]: mode
+ resolution can probe the app db, which a dedicated-only scheduling gate has no business doing."
[]
- (string? (not-empty semantic.db.datasource/db-url)))
+ (semantic.db.datasource/dedicated-url-configured?))
(defn available?
"Whether the entity-retrieval mirror can run right now. All four must hold:
@@ -48,9 +53,11 @@
The feature and config checks can flip at runtime (token/settings entered post-boot), so callers
re-evaluate per use."
[]
- (and (pgvector-configured?)
- (premium-features/has-feature? :library)
+ ;; entitlement first: short-circuit on the license flags for instances that can't use the answer, before
+ ;; the config check (here a cheap dedicated-URL string check, not the app-db pgvector probe)
+ (and (premium-features/has-feature? :library)
(premium-features/has-feature? :library-retrieval)
+ (pgvector-configured?)
(embedding/embedding-supported? (embedding/get-configured-model))))
(defn- index-ready?
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj
index 127137972af..1a08198e44a 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj
@@ -2,6 +2,7 @@
"/api/ee/semantic-search endpoints"
(:require
[clojure.core.memoize :as memoize]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -30,20 +31,22 @@
:indexed_count <number of indexed items>
:total_est <estimated total number of items to index>
- If no index is active, returns an empty map."
+ If no index is active, or no pgvector database is configured, returns an empty map."
[]
(perms/check-has-application-permission :setting)
- (let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)
- active-index (when (and pgvector index-metadata)
- (semantic.index-metadata/get-active-index-state pgvector index-metadata))]
- (try
- (if active-index
- {:indexed_count (active-index-document-count pgvector active-index)
- :total_est (indexible-items-count)}
- {})
- (catch Exception e
- (throw (ex-info "Error fetching semantic search index status" {} e))))))
+ (if-not (semantic.db.datasource/pgvector-configured?)
+ {}
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ active-index (when (and pgvector index-metadata)
+ (semantic.index-metadata/get-active-index-state pgvector index-metadata))]
+ (try
+ (if active-index
+ {:indexed_count (active-index-document-count pgvector active-index)
+ :total_est (indexible-items-count)}
+ {})
+ (catch Exception e
+ (throw (ex-info "Error fetching semantic search index status" {} e)))))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/semantic-search` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 71d065f447e..ca49170a47d 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -4,7 +4,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase-enterprise.semantic-search.embedders]
- [metabase-enterprise.semantic-search.embedding]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
@@ -20,6 +20,8 @@
[potemkin :as p]
[toucan2.realize :as t2.realize]))
+;; import-vars requires full namespace symbols, so it can't use the alias
+#_{:clj-kondo/ignore [:aliased-namespace-symbol]}
(p/import-vars
[metabase-enterprise.semantic-search.embedders
active-embedding-model
@@ -37,7 +39,12 @@
"Enterprise implementation of semantic search engine support check."
:feature :semantic-search
[]
- (semantic.util/semantic-search-available?))
+ ;; Gate engine selection on a usable embedder. App-db mode gives every Postgres instance a pgvector
+ ;; store, so without this the engine auto-activates with no embedder configured and every index and
+ ;; query embed fails. available?/capable? skip the gate on purpose: an existing index still needs
+ ;; maintenance while the embedder is temporarily unconfigured.
+ (and (semantic.util/semantic-search-available?)
+ (semantic.embedding/embedding-supported? (semantic.embedding/get-configured-model))))
(defn build-hnsw-index-async!
"Build the HNSW index on the active semantic search index in the background, returning promptly.
@@ -190,6 +197,7 @@
(init! searchable-documents {}))
(semantic.repair/with-repair-table!
pgvector
+ index-metadata
(fn [repair-table-name]
;; Re-gate all provided documents, populating the repair table as we go
(semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documents
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
index b299d7de5bc..a816698d2d6 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
@@ -2,10 +2,13 @@
(:require
[clojure.string :as str]
[environ.core :refer [env]]
+ [metabase.app-db.core :as mdb]
[metabase.connection-pool :as connection-pool]
[metabase.util :as u]
[metabase.util.log :as log]
- [next.jdbc :as jdbc])
+ [next.jdbc :as jdbc]
+ [next.jdbc.quoted :as quoted]
+ [next.jdbc.result-set :as jdbc.rs])
(:import
(com.mchange.v2.c3p0 DataSources PooledDataSource)
(java.net URLDecoder)
@@ -15,13 +18,30 @@
(set! *warn-on-reflection* true)
(def data-source
- "Atom to hold the pooled JDBC data source for the semantic search database."
+ "Atom to hold the pooled JDBC data source for the semantic search database.
+ Only ever holds a dedicated (MB_PGVECTOR_DB_URL) pool; in app-db mode the shared application pool is
+ returned by [[ensure-initialized-data-source!]] without being stored here, so [[shutdown-db!]] can never
+ close it."
(atom nil))
(def db-url
"The database URL used to connect to pgvector"
(env :mb-pgvector-db-url))
+(defn dedicated-url-configured?
+ "True when MB_PGVECTOR_DB_URL is set to a non-blank value.
+ Canonical check for \"a dedicated pgvector store is configured\"; using it everywhere stops a
+ whitespace-only URL from reading as configured in one subsystem and unset in another."
+ []
+ (not (str/blank? db-url)))
+
+(def app-db-schema
+ "The Postgres schema holding all semantic-search tables when sharing the application database.
+ Isolating by schema makes destructive maintenance (see
+ [[metabase-enterprise.semantic-search.db.migration.impl]]) structurally incapable of touching
+ application tables."
+ "semantic_search")
+
;; All pgvector config — connection *and* connection-pool parameters — rides MB_PGVECTOR_DB_URL, so a
;; deployment sets everything in one place (the DB secret) with no env-var-vs-URL precedence to reason
;; about. c3p0 pool params can't ride the JDBC URL into the driver, so we parse them out here (see
@@ -177,7 +197,151 @@
(init-db!)
(test-connection!))
+(def app-db-pgvector-support
+ "Latches `true` once the app db is confirmed to support pgvector; nil until then.
+ A confirmed extension doesn't vanish under a running instance, so `true` is permanent. A negative result
+ is NOT latched — it opens a [[probe-cooldown-timer]] and is re-probed, so installing pgvector on a
+ running instance is picked up without a restart. Tests reset it."
+ (atom nil))
+
+(def probe-cooldown-timer
+ "A [[u/start-timer]] taken when the last app-db pgvector probe came back unsupported or errored, or nil
+ when a probe is due. Bounds re-probing to once per [[probe-cooldown-ms]] so an absent extension doesn't
+ re-query the catalog on every search and 20s indexer tick. Tests reset it."
+ (atom nil))
+
+(def ^:private probe-cooldown-ms
+ "How long an unsupported or failed app-db pgvector probe is trusted before re-probing.
+ Long enough that a never-provisioned instance isn't running a rolled-back CREATE probe into its DDL audit
+ log every few seconds, short enough to pick up a runtime `CREATE EXTENSION` / privilege grant without a
+ restart."
+ (.toMillis (java.time.Duration/ofMinutes 5)))
+
+(defonce ^{:doc "Log-once latch for the \"no pgvector\" operator hint; the negative probe recurs each
+ cooldown, so without it the hint would repeat. Tests reset it."}
+ logged-pgvector-absent?
+ (atom false))
+
+(defn- probe-due?
+ "True when no app-db pgvector probe cooldown is active."
+ []
+ (let [timer @probe-cooldown-timer]
+ (or (nil? timer) (>= (u/since-ms timer) probe-cooldown-ms))))
+
+(defn- app-db-can-provision-pgvector?
+ "Whether the app-db user can create whichever store pieces are still missing, checked without persisting
+ them: the CREATEs run in a transaction that always rolls back.
+ Attempts CREATE EXTENSION only when `create-extension?` and CREATE SCHEMA only when `create-schema?`, so
+ an already-installed extension or existing schema needs no create privilege.
+ A privilege error reads as false."
+ [app-datasource create-extension? create-schema?]
+ (try
+ (jdbc/with-transaction [tx app-datasource {:rollback-only true}]
+ (when create-extension?
+ (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]))
+ (when create-schema?
+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))])))
+ true
+ (catch Exception e
+ (log/debug e "Semantic search: the application database user cannot provision the pgvector store")
+ false)))
+
+(defn check-app-db-pgvector-support
+ "Can the application database act as the pgvector store?
+ True when the `vector` extension and the [[app-db-schema]] schema are present, or the app-db user can
+ create whichever is missing.
+ Provisioning is verified in a rolled-back transaction (CREATE EXTENSION / CREATE SCHEMA), not read from
+ pg_available_extensions: managed Postgres often lists the extension as available while denying the DDL.
+ The probe persists nothing, so the unlicensed and disabled instances whose availability predicates reach
+ here never mutate the app db; the persisted CREATE EXTENSION / CREATE SCHEMA run only on the activation
+ path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]])."
+ []
+ (let [app-datasource (mdb/data-source)
+ ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not
+ ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog
+ ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to
+ ;; unavailable rather than passing here and crashing later when init creates tables it can't write.
+ {:keys [installed available schema-exists]}
+ (jdbc/execute-one! app-datasource
+ [(str "SELECT"
+ " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"
+ " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"
+ " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")
+ app-db-schema]
+ {:builder-fn jdbc.rs/as-unqualified-kebab-maps})]
+ (cond
+ (not (or installed available)) false
+ (and installed schema-exists) true
+ :else (app-db-can-provision-pgvector? app-datasource
+ (not installed)
+ (not schema-exists)))))
+
+(defn- app-db-pgvector-supported?
+ "Whether the application database can act as the pgvector store, via a cached probe.
+ A confirmed `true` latches for the JVM lifetime (see [[app-db-pgvector-support]]); an unsupported or
+ errored probe is trusted only for [[probe-cooldown-ms]] before re-probing, so a runtime `CREATE
+ EXTENSION` / package install is picked up without a restart while a persistent negative doesn't re-query
+ and re-warn on every call. Returns false while the app db is not yet set up."
+ []
+ (if (true? @app-db-pgvector-support)
+ true
+ (boolean
+ (when (and (mdb/db-is-set-up?) (probe-due?))
+ (locking app-db-pgvector-support
+ (cond
+ (true? @app-db-pgvector-support) true
+ ;; a racing thread probed first and opened the cooldown
+ (not (probe-due?)) false
+ :else
+ (try
+ (if (check-app-db-pgvector-support)
+ (do
+ (log/info (str "Semantic search: using the application database as the pgvector store"
+ " (MB_PGVECTOR_DB_URL is not set). Set it if this instance should use a"
+ " dedicated pgvector database."))
+ (reset! app-db-pgvector-support true)
+ (reset! probe-cooldown-timer nil)
+ true)
+ (do
+ (when (compare-and-set! logged-pgvector-absent? false true)
+ (log/info (str "Semantic search: the application database cannot host pgvector (the"
+ " app-db user cannot create the vector extension or the semantic_search"
+ " schema). Install pgvector and grant CREATE, or set MB_PGVECTOR_DB_URL;"
+ " it is picked up automatically, no restart needed.")))
+ (reset! probe-cooldown-timer (u/start-timer))
+ false))
+ (catch Exception e
+ (reset! probe-cooldown-timer (u/start-timer))
+ (log/warn e (str "Semantic search: pgvector support check on the application database failed;"
+ " will retry after the cooldown."))
+ false))))))))
+
+(defn pgvector-mode
+ "How this instance reaches its pgvector database:
+ :dedicated MB_PGVECTOR_DB_URL is set (always wins)
+ :app-db no URL, but the Postgres application database supports the vector extension
+ :unavailable no pgvector anywhere — semantic search cannot run."
+ []
+ (cond
+ (dedicated-url-configured?) :dedicated
+ (and (= :postgres (mdb/db-type))
+ (app-db-pgvector-supported?)) :app-db
+ :else :unavailable))
+
+(defn pgvector-configured?
+ "Canonical availability predicate: does this instance have a pgvector database to work with?"
+ []
+ (not= :unavailable (pgvector-mode)))
+
(defn ensure-initialized-data-source!
- "Return datasource. Initialize if necessary."
+ "Return datasource. Initialize if necessary.
+ In app-db mode this is the application database's own shared pool."
[]
- (or @data-source (init-db!)))
+ (or @data-source
+ (case (pgvector-mode)
+ :dedicated (init-db!)
+ :app-db (mdb/data-source)
+ :unavailable (throw (ex-info (str "Semantic search requires a pgvector database: set MB_PGVECTOR_DB_URL,"
+ " or use a Postgres application database with the pgvector extension"
+ " available.")
+ {:mode :unavailable})))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
index 5a8769b04ac..f7f91748932 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
@@ -3,6 +3,7 @@
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
[metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.util.log :as log]
[next.jdbc :as jdbc])
(:import
@@ -14,17 +15,39 @@
[:migrated_at :timestamp [:default [:NOW]]]
[:status [:varchar 32]]])
-(def ^:private migration-table-hsql
- (-> (sql.helpers/create-table :migration :if-not-exists)
- (sql.helpers/with-columns columns)))
+(defn- migration-table-kw
+ "The migration bookkeeping table: inside the module's schema when index-metadata carries one (shared
+ app-db mode), bare `migration` otherwise."
+ [index-metadata]
+ (if-let [schema (:schema index-metadata)]
+ (keyword (str schema ".migration"))
+ :migration))
+
+(defn- ensure-schema-exists!
+ "Create the module's schema when index-metadata carries one.
+ This is the first *persisted* write against the app db in app-db mode; insufficient privileges normally
+ surface earlier, at the capability probe's rolled-back CREATE SCHEMA."
+ [index-metadata tx]
+ (when-let [schema (:schema index-metadata)]
+ (try
+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (semantic.util/quote-ident schema))])
+ (catch SQLException e
+ (throw (ex-info (format (str "Failed to create the %s schema on the application database. Grant"
+ " CREATE on the database to the Metabase user, or set"
+ " MB_PGVECTOR_DB_URL to use a dedicated pgvector database.")
+ schema)
+ {:type ::schema-creation-failed :schema schema}
+ e))))))
(defn- migration-table-sql
- []
- (sql/format migration-table-hsql))
+ [index-metadata]
+ (-> (sql.helpers/create-table (migration-table-kw index-metadata) :if-not-exists)
+ (sql.helpers/with-columns columns)
+ (sql/format)))
(defn- ensure-migration-table!
- [tx]
- (let [sql (migration-table-sql)]
+ [index-metadata tx]
+ (let [sql (migration-table-sql index-metadata)]
(try
(jdbc/execute! tx sql)
(catch SQLException e
@@ -35,23 +58,23 @@
(defn- db-version
"Get database version of schema from migration table."
- [tx]
+ [index-metadata tx]
(or (:max_version
(jdbc/execute-one! tx
(sql/format {:select [[[:max :version] :max_version]]
- :from [:migration]})))
+ :from [(migration-table-kw index-metadata)]})))
-1))
(defn- write-successful-migration!
- [tx]
- (jdbc/execute-one! tx (sql/format (-> (sql.helpers/insert-into :migration)
+ [index-metadata tx]
+ (jdbc/execute-one! tx (sql/format (-> (sql.helpers/insert-into (migration-table-kw index-metadata))
(sql.helpers/values [{:version semantic.db.migration.impl/schema-version
:status "success"}])))))
(defn maybe-migrate-schema!
"Execute schema migration (control, meta, gate, ...) if appropriate."
- [tx opts]
- (let [db-version (db-version tx)]
+ [tx {:keys [index-metadata] :as opts}]
+ (let [db-version (db-version index-metadata tx)]
(cond
(= db-version semantic.db.migration.impl/schema-version)
(log/info "Migration already performed, skipping.")
@@ -61,7 +84,7 @@
(log/infof "Starting migration from version %d to %d."
db-version semantic.db.migration.impl/schema-version)
(semantic.db.migration.impl/migrate-schema! tx opts)
- (write-successful-migration! tx))
+ (write-successful-migration! index-metadata tx))
:else
(log/infof "Database schema version (%d) is newer than code version (%d). Not performing migration."
@@ -69,27 +92,23 @@
nil)
(defn- index-metadata-table-exists?
- [tx]
- (jdbc/execute-one! tx
- (sql/format {:select [[[:raw 1] :exists]]
- :from [:information_schema.tables]
- :where [[:= :information_schema.tables.table_name [:inline "index_metadata"]]]
- :limit 1})))
+ [index-metadata tx]
+ (semantic.util/table-exists? tx (:metadata-table-name index-metadata)))
(defn- lowest-dynamic-db-version
"Lowest index version. If there is lower than defined in code dynamic schema migration will be attempted."
- [tx]
- (or (when (index-metadata-table-exists? tx)
+ [index-metadata tx]
+ (or (when (index-metadata-table-exists? index-metadata tx)
(:min_index
(jdbc/execute-one! tx
(sql/format {:select [[[:min :index_version] :min_index]]
- :from [:index_metadata]}))))
+ :from [(keyword (:metadata-table-name index-metadata))]}))))
0))
(defn maybe-migrate-dynamic-schema!
"Migration for dynamic tables (index_table_xyzs) if appropriate."
- [tx opts]
- (let [db-version (lowest-dynamic-db-version tx)]
+ [tx {:keys [index-metadata] :as opts}]
+ (let [db-version (lowest-dynamic-db-version index-metadata tx)]
(cond
(= db-version semantic.db.migration.impl/dynamic-schema-version)
(log/info "Dynamic tables migration already performed, skipping.")
@@ -107,13 +126,14 @@
(defn maybe-migrate!
"Execute schema and dynamic schema migrations."
- [tx opts]
- (ensure-migration-table! tx)
+ [tx {:keys [index-metadata] :as opts}]
+ (ensure-schema-exists! index-metadata tx)
+ (ensure-migration-table! index-metadata tx)
(maybe-migrate-schema! tx opts)
(maybe-migrate-dynamic-schema! tx opts)
nil)
(defn drop-migration-table!
"Drop migration table."
- [connectable]
- (jdbc/execute! connectable (sql/format (sql.helpers/drop-table :if-exists :migration))))
+ [index-metadata connectable]
+ (jdbc/execute! connectable (sql/format (sql.helpers/drop-table :if-exists (migration-table-kw index-metadata)))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index ca535fb0c50..c560223c7ce 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -2,11 +2,13 @@
(:require
[honey.sql :as sql]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.collections.curation :as collections.curation]
[metabase.config.core :as config]
[metabase.util :as u]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]
[toucan2.core :as t2]))
(def schema-version
@@ -14,28 +16,53 @@
schema migration will be performed."
2)
+(def ^:private app-db-sentinel-tables
+ "Tables specific to a Metabase application database, chosen to avoid generic names (e.g. Liquibase's
+ `databasechangelog`) that a genuinely dedicated pgvector database might also carry.
+ Their presence in a to-be-wiped dedicated database means MB_PGVECTOR_DB_URL was pointed at an app db."
+ #{"core_user" "metabase_database"})
+
(defn- drop-all-but-migration-table
- [tx]
- (let [table-names (map (comp first vals)
- (jdbc/execute! tx
- (sql/format
- {:select [[:tablename :xi]]
- :from [:pg_tables]
- :where [:and
- [:<> :schemaname [:inline "information_schema"]]
- [:<> :schemaname [:inline "pg_catalog"]]
- [:<> :tablename [:inline "migration"]]]})))]
- (doseq [table table-names]
+ "Destructive: clears out semantic-search storage ahead of recreating it from scratch.
+ When index-metadata carries a `:schema` (shared app-db mode) ONLY tables inside that schema may be
+ dropped — the application's tables live in other schemas and must never be touched here.
+ Without a `:schema` the database is assumed dedicated to semantic search and its default schema is
+ wiped; refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed at
+ the application database would otherwise destroy it here, on first init)."
+ [index-metadata tx]
+ (let [schema (:schema index-metadata)
+ tables (jdbc/execute! tx
+ (sql/format
+ {:select [:schemaname :tablename]
+ :from [:pg_tables]
+ :where (if schema
+ [:and
+ [:= :schemaname [:inline schema]]
+ [:<> :tablename [:inline "migration"]]]
+ ;; dedicated mode only ever creates tables in the default
+ ;; schema, so the reset has no business outside it
+ [:and
+ [:= :schemaname [:raw "current_schema()"]]
+ [:<> :tablename [:inline "migration"]]])})
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (when (and (nil? schema)
+ (some (comp app-db-sentinel-tables :tablename) tables))
+ (throw (ex-info (str "Refusing to reset the semantic search database: it contains Metabase application"
+ " tables. Point MB_PGVECTOR_DB_URL at a dedicated pgvector database, or unset it to"
+ " share the application database in the isolated semantic_search schema.")
+ {:type ::refused-app-db-wipe})))
+ (doseq [{:keys [schemaname tablename]} tables]
(jdbc/execute! tx
(sql/format
- {:drop-table [[[:raw table]]]})))))
+ {:drop-table [[[:raw (str (semantic.util/quote-ident schemaname) "."
+ (semantic.util/quote-ident tablename))]]]})))))
(defn migrate-schema!
"Migrate schema (control, metadata, gate, ...). Migration author is responsible for removing leftovers if necessary
and in general leaving schema in desired state."
[tx {:keys [index-metadata] :as _opts}]
;; ideally index_table indexed are manipulated in dynamic schema part but for now it does not matter
- (drop-all-but-migration-table tx)
+ (drop-all-but-migration-table index-metadata tx)
(semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)
(semantic.index-metadata/ensure-control-row-exists! tx index-metadata))
@@ -51,18 +78,21 @@
in place to be cleaned up or re-created elsewhere."
[tx index-metadata target-version alter-fn]
(let [metadata-table (keyword (:metadata-table-name index-metadata))
+ schema (:schema index-metadata)
execute! (fn [q] (jdbc/execute! tx (sql/format q)))
candidate-names (->> (execute! {:select-distinct [:table_name]
:from [metadata-table]
:where [[:< :index_version target-version]]})
(mapcat vals))
- existing-names (when (seq candidate-names)
+ ;; stored table_name values are schema-qualified in app-db mode; pg_tables lists bare names
+ existing-bare (when (seq candidate-names)
(->> (execute! {:select [:tablename]
:from [:pg_tables]
- :where [:in :tablename (vec candidate-names)]})
+ :where (cond-> [:and [:in :tablename (mapv semantic.util/table-name-part candidate-names)]]
+ schema (conj [:= :schemaname [:inline schema]]))})
(mapcat vals)
set))
- table-names (filter existing-names candidate-names)]
+ table-names (filter (comp existing-bare semantic.util/table-name-part) candidate-names)]
(when (seq table-names)
(doseq [table-name table-names]
(alter-fn execute! table-name))
@@ -122,12 +152,12 @@
(fn [execute! table-name]
(let [kw-tbl (keyword table-name)
kw-gate (keyword gate-table)
- tbl-model (keyword table-name "model")
- tbl-model-id (keyword table-name "model_id")
- tbl-root-coll-type (keyword table-name "root_collection_type")
- gate-id (keyword gate-table "id")
- gate-doc-root [:->> (keyword gate-table "document") [:inline "root_collection_type"]]
- composite-gate-id [:|| tbl-model [:inline "_"] tbl-model-id]]
+ model-col (semantic.util/column-keyword table-name "model")
+ model-id-col (semantic.util/column-keyword table-name "model_id")
+ root-coll-type-col (semantic.util/column-keyword table-name "root_collection_type")
+ gate-id (semantic.util/column-keyword gate-table "id")
+ gate-doc-root [:->> (semantic.util/column-keyword gate-table "document") [:inline "root_collection_type"]]
+ composite-gate-id [:|| model-col [:inline "_"] model-id-col]]
(execute! {:alter-table [kw-tbl] :add-column [[:root_collection_type :text :if-not-exists]]})
;; Per-row backfill: take whatever the gate document says — authoritative when present.
(execute! {:update kw-tbl
@@ -135,7 +165,7 @@
:set {:root_collection_type gate-doc-root}
:where [:and
[:= gate-id composite-gate-id]
- [:= tbl-root-coll-type nil]
+ [:= root-coll-type-col nil]
[:!= gate-doc-root nil]]})
;; Forest backfill: one UPDATE per distinct root type, filling rows the gate doc missed.
(doseq [[root-type entries] (group-by val root-type-by-coll-id)]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj
index 0ec0f9bcdf1..b3e2702cc89 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj
@@ -67,8 +67,9 @@
sql (sql/format ddl :quoted true)]
(jdbc/execute-one! pgvector sql))
;; create attempt_at index
- (let [ddl {:create-index [[(keyword (str (name dlq-table) "_attempt_at_idx")) :if-not-exists] [dlq-table :attempt_at]]}
- sql (sql/format ddl :quoted true)]
+ (let [idx-name (keyword (str (semantic.util/table-name-part (name dlq-table)) "_attempt_at_idx"))
+ ddl {:create-index [[idx-name :if-not-exists] [dlq-table :attempt_at]]}
+ sql (sql/format ddl :quoted true)]
(jdbc/execute-one! pgvector sql))
nil))
@@ -191,7 +192,9 @@
[pgvector index-metadata index-id dlq-entries]
(if (empty? dlq-entries)
0
- (let [dml {:insert-into (dlq-table-name-kw index-metadata index-id)
+ (let [dlq-table (dlq-table-name-kw index-metadata index-id)
+ dlq-col (semantic.util/conflict-target-column (name dlq-table) "error_gated_at")
+ dml {:insert-into dlq-table
:values (sort-by :gate_id dlq-entries)
:on-conflict [:gate_id]
:do-update-set {:fields {:attempt_at :excluded.attempt_at
@@ -200,8 +203,7 @@
:last_attempted_at :excluded.last_attempted_at}
;; condition: if exiting dlq entry reflects a more recent gate failure
;; prefer the new attempt_at / retry_count from the gate to the prior DLQ entry.
- :where [:<= [:. (dlq-table-name-kw index-metadata index-id) :error_gated_at]
- :excluded.error_gated_at]}}
+ :where [:<= dlq-col :excluded.error_gated_at]}}
sql (sql/format dml :quoted true)
{upsert-count ::jdbc/update-count} (jdbc/execute-one! pgvector sql)]
(log/debugf "Added/updated %d DLQ entries for index %s" upsert-count index-id)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml b/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml
index e01038b6f3c..a3083464ed7 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml
@@ -1,3 +1,8 @@
+# Dev harness for a DEDICATED pgvector database (MB_PGVECTOR_DB_URL). A dedicated database is not
+# required: when MB_PGVECTOR_DB_URL is unset and the application database is Postgres with the pgvector
+# extension available, semantic search automatically runs against the app db — its tables live in a
+# dedicated `semantic_search` schema and share the app-db connection pool (so heavy reindexing competes
+# with regular app-db traffic; use a dedicated database when that matters).
version: '3.8'
services:
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj
index ef08ffc6db7..ec73a6e6368 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj
@@ -21,7 +21,10 @@
(defn get-index-metadata
"Returns this instances index metadata configuration.
- Currently, an instance only has one metadata root, so this will always have the same value in production."
+ Currently, an instance only has one metadata root, so this will always have the same value in production:
+ the schema-qualified configuration when sharing the application database, the bare default otherwise."
[]
;; still useful to indirect this for tests
- semantic.index-metadata/default-index-metadata)
+ (if (= :app-db (semantic.db.datasource/pgvector-mode))
+ semantic.index-metadata/app-db-index-metadata
+ semantic.index-metadata/default-index-metadata))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj
index e8e4e6ecb5d..434bb682245 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj
@@ -22,6 +22,7 @@
[buddy.core.hash :as buddy-hash]
[honey.sql :as sql]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
[metabase.util :as u]
[metabase.util.json :as json]
@@ -122,6 +123,7 @@
(peek updates)))
seq)]
(let [{:keys [gate-table-name]} index-metadata
+ gate-col (fn [column] (semantic.util/conflict-target-column gate-table-name column))
upsert-q {:insert-into (keyword gate-table-name)
;; sort to ensure locks are acquired predictably
:values (sort-by :id gate-document-batch)
@@ -134,18 +136,18 @@
:gated_at [:clock_timestamp]}
:where [:and
;; in the case of an updated_at collision, simulate old behaviour, later transactions on the gate win.
- [:<= (keyword gate-table-name "updated_at") :excluded.updated_at]
+ [:<= (gate-col "updated_at") :excluded.updated_at]
;; content diff
[:case
;; now deleted, delete if was not previously
[:= nil :excluded.document_hash]
- [:!= nil (keyword gate-table-name "document_hash")]
+ [:!= nil (gate-col "document_hash")]
;; was deleted (now has a value) - update
- [:= nil (keyword gate-table-name "document_hash")]
+ [:= nil (gate-col "document_hash")]
true
;; update if new value is different
:else
- [:!= (keyword gate-table-name "document_hash") :excluded.document_hash]]]}}
+ [:!= (gate-col "document_hash") :excluded.document_hash]]]}}
upsert-sql (sql/format upsert-q :quoted true)]
(jdbc/execute! tx [(format "SET LOCAL statement_timeout = %d" (.toMillis gate-write-timeout))]) ; note pg cannot accept a parameter here
(let [stmt-start (u/start-timer)]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index b51277fface..6f267624373 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -440,9 +440,17 @@
(defn- index-name
"Returns the name for an index for the given index configuration, column, and index type."
[index suffix]
- (let [index-name (str (:table-name index) suffix)]
+ ;; index names are bare: an index lands in its table's schema and cannot be schema-qualified
+ (let [index-name (str (semantic.util/table-name-part (:table-name index)) suffix)]
(hash-identifier-if-exceeds-pg-limit index-name)))
+(defn schema-qualified-index-name
+ "The index name qualified with its table's schema (when the table has one), for catalog lookups —
+ an index lives in the same schema as its table."
+ [index index-name]
+ (let [[schema _] (semantic.util/qualified-table-parts (:table-name index))]
+ (cond->> index-name schema (str schema "."))))
+
(defn hnsw-index-name
"Returns the name for a HNSW database index for the given semantic search index configuration."
[index]
@@ -503,7 +511,15 @@
(format "whitespace in the table name (%s) is not currently supported" table-name))
{:keys [vector-dimensions]} embedding-model]
(log/info "Creating index table" table-name)
- (jdbc/execute! connectable (sql/format (sql.helpers/create-extension :vector :if-not-exists)))
+ (try
+ (jdbc/execute! connectable (sql/format (sql.helpers/create-extension :vector :if-not-exists)))
+ (catch Exception e
+ ;; Extension install commonly fails on privileges (may need superuser); give the operator the way out.
+ (throw (ex-info (str "Failed to install the pgvector extension. Have a privileged user run"
+ " CREATE EXTENSION vector; on this database, or set MB_PGVECTOR_DB_URL to a"
+ " database where it is installed.")
+ {:type ::extension-install-failed}
+ e))))
(when force-reset? (drop-index-table! connectable index))
(jdbc/execute!
connectable
@@ -532,7 +548,10 @@
[(keyword table-name) :content])
sql-format-quoted)))
(catch Exception e
- (throw (ex-info "Failed to create index table" {} e)))))
+ ;; let an already-actionable error (e.g. the pgvector-extension guidance) surface unwrapped
+ (if (= ::extension-install-failed (:type (ex-data e)))
+ (throw e)
+ (throw (ex-info "Failed to create index table" {} e))))))
(comment
(def embedding-model {:provider "ollama"
@@ -1031,13 +1050,17 @@
(first decoded)))
(defn- find-scan-node
- "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`."
+ "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`.
+ EXPLAIN reports \"Relation Name\" unqualified, so compare on the bare name part."
[plan table-name]
- (when (map? plan)
- (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get plan "Node Type"))
- (= table-name (get plan "Relation Name")))
- plan
- (some #(find-scan-node % table-name) (get plan "Plans")))))
+ (let [bare-name (semantic.util/table-name-part table-name)
+ search (fn search [node]
+ (when (map? node)
+ (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get node "Node Type"))
+ (= bare-name (get node "Relation Name")))
+ node
+ (some search (get node "Plans")))))]
+ (search plan)))
(defn- scan-node-metrics
"Pull the index-table scan node out of an EXPLAIN plan tree and summarise it: the plan node chosen, the
@@ -1134,7 +1157,7 @@
;; path); production traffic leaves it unset and gets the fail-fast.
(when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))
(not (:vector-search-allow-missing-index? search-context))
- (not (semantic.util/index-exists? db (hnsw-index-name index))))
+ (not (semantic.util/index-exists? db (schema-qualified-index-name index (hnsw-index-name index)))))
(throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. "
"Set the semantic-search-vector-strategy setting to an index-backed strategy "
"(:hnsw or :hnsw-iterative-*) to build it.")
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
index ad3736e86b6..67aa335c3de 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
@@ -8,6 +8,7 @@
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
;; TODO: extract schema code to go under db.migration
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.util :as u]
@@ -73,13 +74,29 @@
(update :table-name qualify))))
(def default-index-metadata
- "The default index metadata configuration that will be used for the search engine integration."
+ "The index metadata configuration for a dedicated pgvector database (MB_PGVECTOR_DB_URL)."
{:version "2"
:metadata-table-name "index_metadata"
:control-table-name "index_control"
:gate-table-name "index_gate"
:index-table-qualifier "%s"})
+(def app-db-index-metadata
+ "The index metadata configuration used when semantic search shares the application database.
+ Every table lives inside a dedicated schema, so destructive maintenance (see
+ [[metabase-enterprise.semantic-search.db.migration.impl]]) is structurally incapable of touching
+ application tables.
+ Table names are stored and threaded around schema-qualified; derive identifiers and query catalogs via
+ [[metabase-enterprise.semantic-search.util/table-name-part]]."
+ ;; HoneySQL renders the dotted keywords as schema-qualified identifiers, quoted or not.
+ (let [schema semantic.db.datasource/app-db-schema]
+ {:version "2"
+ :schema schema
+ :metadata-table-name (str schema ".index_metadata")
+ :control-table-name (str schema ".index_control")
+ :gate-table-name (str schema ".index_gate")
+ :index-table-qualifier (str schema ".%s")}))
+
(defn- create-index-metadata-table-if-not-exists-sql [index-metadata]
(let [{:keys [metadata-table-name]} index-metadata
schema (:metadata schema-0)]
@@ -151,7 +168,7 @@
pgvector
(sql/format
(sql.helpers/create-index
- [(keyword (str (:gate-table-name index-metadata) "_gated_at")) :if-not-exists]
+ [(keyword (str (semantic.util/table-name-part (:gate-table-name index-metadata)) "_gated_at")) :if-not-exists]
[(keyword (:gate-table-name index-metadata)) :gated_at :id])
:quoted true))
(log/info "Creating gate table tombstone cleanup index if not exists")
@@ -160,7 +177,7 @@
pgvector
(sql/format
{:create-index
- [[(keyword (str (:gate-table-name index-metadata) "_tombstone_cleanup")) :if-not-exists]
+ [[(keyword (str (semantic.util/table-name-part (:gate-table-name index-metadata)) "_tombstone_cleanup")) :if-not-exists]
[(keyword (:gate-table-name index-metadata)) :gated_at]]
:where [:and [:= :document nil] [:= :document_hash nil]]}
:quoted true))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
index eec0e025430..da233f0d664 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
@@ -11,6 +11,7 @@
[honey.sql.helpers :as sql.helpers]
[java-time.api :as t]
[medley.core :as m]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.string :as u.str]
@@ -40,20 +41,24 @@
but are not in the repair table. These represent lost deletes."
[pgvector gate-table-name repair-table-name]
(try
- (let [anti-join-sql (-> (sql.helpers/select :model :model_id)
+ (let [column semantic.util/column-keyword
+ anti-join-sql (-> (sql.helpers/select :model :model_id)
(sql.helpers/from (keyword gate-table-name))
(sql.helpers/where [:not [:exists
(-> (sql.helpers/select 1)
(sql.helpers/from (keyword repair-table-name))
(sql.helpers/where [:and
- [:= (keyword repair-table-name "model")
- (keyword gate-table-name "model")]
- [:= (keyword repair-table-name "model_id")
- (keyword gate-table-name "model_id")]]))]])
+ [:= (column repair-table-name "model")
+ (column gate-table-name "model")]
+ [:= (column repair-table-name "model_id")
+ (column gate-table-name "model_id")]]))]])
(sql/format :quoted true))
results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
(log/infof "Found %d documents in gate table that should be deleted" (count results))
results)
+ ;; TODO (Chris 2026-07-13) -- swallowing the error reads as "no lost deletes", so a transient anti-join
+ ;; failure silently skips tombstoning that run. Tolerated because the next repair pass retries; revisit
+ ;; if lost deletes start slipping through.
(catch Exception e
(log/errorf e "Error finding lost deletes between gate table %s and repair table %s"
gate-table-name repair-table-name))))
@@ -89,18 +94,20 @@
(log/warnf e "Failed to drop repair table: %s" repair-table-name))))
(defn- repair-table-name
- "Generates a unique name for a repair table with timestamp for cleanup.
+ "Generates a unique name for a repair table with timestamp for cleanup, qualified like the index tables
+ so it lands in the module's schema when sharing the app db.
Format: repair_<millis-since-epoch>_<short-id>"
- []
+ [index-metadata]
(let [millis-since-epoch (t/to-millis-from-epoch (t/instant))
short-id (u/lower-case-en (u.str/random-string 6))]
- (format "repair_%d_%s" millis-since-epoch short-id)))
+ (format (:index-table-qualifier index-metadata "%s")
+ (format "repair_%d_%s" millis-since-epoch short-id))))
(defn with-repair-table!
"Creates a repair table, executes a function with the table name, and ensures cleanup.
Returns the result of calling f with the repair table name."
- [pgvector f]
- (let [repair-table (repair-table-name)]
+ [pgvector index-metadata f]
+ (let [repair-table (repair-table-name index-metadata)]
(try
(create-repair-table! pgvector repair-table)
(f repair-table)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
index f382d11680b..d684057d4b6 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
@@ -21,21 +21,48 @@
(set! *warn-on-reflection* true)
+;; Both orphan sweeps feed table names to DROP, so in shared app-db mode the schema scoping keeps a LIKE
+;; pattern from matching and dropping application tables.
+
+(defn- scope-where-to-schema
+ "Restrict an `information_schema.tables` WHERE (an aliased-`:t` `[:and ...]` vector) to the module schema.
+ A no-op without a schema (dedicated mode)."
+ [where schema]
+ (cond-> where schema (conj [:= :t.table_schema [:inline schema]])))
+
+(defn- requalify-table-names
+ "Prefix bare `information_schema` table names with the module schema so they match how the metadata table
+ stores them. A no-op in dedicated mode."
+ [schema table-names]
+ (map #(cond->> % schema (str schema ".")) table-names))
+
(defn- orphan-index-tables
"Returns a list of semantic search index tables which are not referenced in the metadata table.
- Not expected to occur, but if it does, these tables can be dropped."
- [pgvector {:keys [metadata-table-name]}]
- (let [orphaned-tables-sql
+ Not expected to occur, but if it does, these tables can be dropped.
+ With a `:schema` (shared app-db mode) only tables inside the module's schema are considered, and the
+ returned names are schema-qualified to match how the metadata table stores them."
+ ;; TODO (Chris 2026-07-09) -- BOT-1833: the LIKE 'index_table_%' pattern predates the
+ ;; index_<provider>_<model>_<dims> naming, so it matches nothing and orphan detection is a no-op. A fix
+ ;; needs shape-aware matching that excludes index_metadata/index_control/index_gate (not registered in
+ ;; metadata.table_name — a naive 'index\_%' pattern would drop them as orphans).
+ [pgvector {:keys [metadata-table-name schema]}]
+ (let [;; information_schema reports bare names; stored metadata names are qualified when we have a schema
+ stored-name (if schema
+ [:|| [:inline (str schema ".")] :t.table_name]
+ :t.table_name)
+ orphaned-tables-sql
(-> {:select [:t.table_name]
:from [[:information_schema.tables :t]]
:left-join [[(keyword metadata-table-name) :meta]
- [:= :meta.table_name :t.table_name]]
- :where [:and
- [:like :t.table_name [:inline "index_table_%"]]
- [:= :meta.table_name nil]]}
+ [:= :meta.table_name stored-name]]
+ :where (scope-where-to-schema [:and
+ [:like :t.table_name [:inline "index_table_%"]]
+ [:= :meta.table_name nil]]
+ schema)}
(sql/format :quoted true))]
(->> (jdbc/execute! pgvector orphaned-tables-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (map :table_name))))
+ (map :table_name)
+ (requalify-table-names schema))))
(defn- parse-repair-table-timestamp
"Extracts timestamp from repair table name. Returns nil if parsing fails."
@@ -51,19 +78,22 @@
Repair tables are named: repair_<millis-since-epoch>_<short-id>
Repair tables should not become orphaned under normal operation, since they should be deleted immeditaely after use.
- However, in case of a crash or failure, they may be left behind, so we clean them up after a retention period."
- [pgvector]
+ However, in case of a crash or failure, they may be left behind, so we clean them up after a retention period.
+ With a `:schema` (shared app-db mode) only tables inside the module's schema are considered — a `repair_%`
+ pattern must never match application tables — and the returned names are schema-qualified."
+ [pgvector {:keys [schema]}]
(let [retention-cutoff (t/minus (t/instant) (t/hours (semantic.settings/repair-table-retention-hours)))
repair-tables-sql (-> {:select [:t.table_name]
:from [[:information_schema.tables :t]]
- :where [:like :t.table_name [:inline "repair_%"]]}
+ :where (scope-where-to-schema [:and [:like :t.table_name [:inline "repair_%"]]] schema)}
(sql/format :quoted true))
all-repair-tables (->> (jdbc/execute! pgvector repair-tables-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})
(map :table_name))
- old-tables (filter (fn [table-name]
- (when-let [table-timestamp (parse-repair-table-timestamp table-name)]
- (t/before? table-timestamp retention-cutoff)))
- all-repair-tables)]
+ old-tables (->> all-repair-tables
+ (filter (fn [table-name]
+ (when-let [table-timestamp (parse-repair-table-timestamp table-name)]
+ (t/before? table-timestamp retention-cutoff))))
+ (requalify-table-names schema))]
(when (seq old-tables)
(log/infof "Found %d orphaned repair tables older than %d hours"
(count old-tables)
@@ -72,8 +102,8 @@
(defn- cleanup-orphan-repair-tables!
"Cleans up repair tables that are older than the retention period."
- [pgvector]
- (let [orphan-tables (orphan-repair-tables pgvector)]
+ [pgvector index-metadata]
+ (let [orphan-tables (orphan-repair-tables pgvector index-metadata)]
(when (seq orphan-tables)
(let [tables-to-drop (map keyword orphan-tables)
drop-table-sql (sql/format
@@ -168,13 +198,17 @@
(when (semantic.u/semantic-search-available?)
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.env/get-index-metadata)]
- ;; Available but never initialized is a steady state (semantic neither default nor additional);
- ;; the bookkeeping tables only exist after init, and the cleanup queries would throw without them.
+ ;; Available but never initialized is a steady state (semantic neither default nor additional,
+ ;; or app-db mode with the engine disabled); the bookkeeping tables only exist after init, and
+ ;; the cleanup queries would throw without them.
;; Still runs while merely available, so an opted-out instance's leftover indexes get cleaned.
(when (semantic.index-metadata/control-and-metadata-tables-exist? pgvector index-metadata)
(cleanup-stale-indexes! pgvector index-metadata)
- (cleanup-old-gate-tombstones! pgvector index-metadata)
- (cleanup-orphan-repair-tables! pgvector)))))
+ (cleanup-old-gate-tombstones! pgvector index-metadata))
+ ;; Repair-table cleanup is information_schema-driven, so it needs no bookkeeping tables and runs
+ ;; outside the guard above. Shared app-db mode scopes the sweep to the module schema; dedicated mode
+ ;; owns the whole DB, so its `repair_%` sweep is DB-wide by design (a mispointed URL is the caller's).
+ (cleanup-orphan-repair-tables! pgvector index-metadata))))
(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))
(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index a0c9c733650..09452dd8efe 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -51,9 +51,11 @@
;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed
;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY
;; registers the index in pg_indexes as soon as the build starts, so this fires only once.
- (when (and (hnsw-strategy?)
- (not (semantic.u/index-exists? pgvector (semantic.index/hnsw-index-name index))))
- (semantic.core/build-hnsw-index-async!))
+ (let [hnsw-index (semantic.index/schema-qualified-index-name
+ index (semantic.index/hnsw-index-name index))]
+ (when (and (hnsw-strategy?)
+ (not (semantic.u/index-exists? pgvector hnsw-index)))
+ (semantic.core/build-hnsw-index-async!)))
(semantic-search.indexer/quartz-job-run! pgvector index-metadata))
;; Engines can activate at runtime (license applied, kill switch re-enabled,
;; additional-search-engines set on another node); initializing from the next tick heals
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 69fe3590a3b..055c32d6a66 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -1,46 +1,97 @@
(ns metabase-enterprise.semantic-search.util
(:require
+ [clojure.string :as str]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase.app-db.core :as mdb]
[metabase.premium-features.core :as premium-features]
[metabase.search.engine :as search.engine]
[next.jdbc :as jdbc]
+ [next.jdbc.quoted :as quoted]
[next.jdbc.result-set :as jdbc.rs]))
+(def quote-ident
+ "Quote a Postgres identifier for interpolation into raw SQL, doubling any embedded double quote."
+ quoted/postgres)
+
+(defn qualified-table-parts
+ "Split a possibly schema-qualified table name into `[schema table]`; `schema` is nil when unqualified."
+ [table-name]
+ (let [[a b] (str/split table-name #"\." 2)]
+ (if b [a b] [nil a])))
+
+(defn table-name-part
+ "The bare table name of a possibly schema-qualified table name.
+ Derived identifiers (index names, catalog lookups by `tablename`) must use this, never the full name."
+ [table-name]
+ (second (qualified-table-parts table-name)))
+
+(defn column-keyword
+ "A `table.column` reference as a dotted-name keyword, not a namespaced one.
+ HoneySQL quotes a keyword's namespace as one identifier, so a schema-qualified table there renders as a
+ single broken identifier. The dotted name renders as separate quoted parts.
+ Pass the table name as it should render: qualified for an ordinary FROM/JOIN reference."
+ [table-name column]
+ (keyword (str table-name "." (name column))))
+
+(defn conflict-target-column
+ "A `column-keyword` for an ON CONFLICT clause on a possibly schema-qualified `table-name`.
+ Postgres names the conflict target by its bare relation even when the insert target is schema-qualified,
+ so the reference drops the schema."
+ [table-name column]
+ (column-keyword (table-name-part table-name) column))
+
(defn table-exists?
- "Does the table-name exist in pgvector DB's information_schema.tables?"
+ "Does the table-name exist in pgvector DB's information_schema.tables?
+ A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."
[pgvector table-name]
- (-> (jdbc/execute-one! pgvector
- ["SELECT exists (select 1 FROM information_schema.tables WHERE table_name = ?) table_exists"
- table-name]
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (:table_exists false)))
+ (let [[schema table] (qualified-table-parts table-name)]
+ (-> (jdbc/execute-one! pgvector
+ (if schema
+ [(str "SELECT exists (select 1 FROM information_schema.tables"
+ " WHERE table_schema = ? AND table_name = ?) table_exists")
+ schema table]
+ ["SELECT exists (select 1 FROM information_schema.tables WHERE table_name = ?) table_exists"
+ table])
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (:table_exists false))))
(defn index-exists?
- "Does an index named `index-name` exist in the pgvector DB's pg_indexes?"
+ "Does an index named `index-name` exist in the pgvector DB's pg_indexes?
+ A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."
[pgvector index-name]
- (-> (jdbc/execute-one! pgvector
- ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"
- index-name]
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (:index_exists false)))
+ (let [[schema index] (qualified-table-parts index-name)]
+ (-> (jdbc/execute-one! pgvector
+ (if schema
+ ["SELECT exists (select 1 FROM pg_indexes WHERE schemaname = ? AND indexname = ?) index_exists"
+ schema index]
+ ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"
+ index])
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (:index_exists false))))
(defn semantic-search-configured?
- "Is a pgvector DB configured for this instance?
- The boot-static input: gates Quartz job scheduling, which happens once at startup.
- The runtime gates (license, kill switch, engine activity) are checked per job execution instead, so
- flipping them never requires a restart.
- This check becomes a DB probe under BOT-1796."
+ "Whether to schedule the semantic-search Quartz jobs at startup.
+ True when the `:semantic-search` feature is present and a pgvector store might exist: a dedicated
+ MB_PGVECTOR_DB_URL, or a Postgres app DB that [[semantic-search-capable?]] can probe to answer for sure.
+ Cheap and infallible by contract -- it runs at boot and never queries the DB."
[]
- (string? (not-empty semantic.db.datasource/db-url)))
+ ;; The license is in this boot gate, not only the per-execution gates, so an unlicensed instance's
+ ;; scheduler stays free of no-op jobs. The asymmetry is deliberate: removing the feature at runtime lets
+ ;; the scheduled jobs no-op via semantic-search-active?, but adding it needs a restart before they
+ ;; schedule. The kill switch and engine activity stay per-execution so they never need one.
+ (and (premium-features/has-feature? :semantic-search)
+ (or (semantic.db.datasource/dedicated-url-configured?)
+ (= :postgres (mdb/db-type)))))
(defn semantic-search-capable?
"Does this instance have the infrastructure for semantic search: the premium feature and a pgvector DB.
Deliberately excludes the kill switch, which is checked per execution so it works at runtime."
[]
- ;; Cheap gate first: semantic-search-configured? becomes a DB probe under BOT-1796.
+ ;; Feature first: the pgvector check may probe the app DB, and instances that can't use the answer
+ ;; must never probe.
(and (premium-features/has-feature? :semantic-search)
- (semantic-search-configured?)))
+ (semantic.db.datasource/pgvector-configured?)))
(defn semantic-search-available?
"Whether semantic search can run on this instance: capable and not disabled by the kill switch.
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
index 7f7ea85b3ff..472d0f20064 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
@@ -10,10 +10,15 @@
[metabase.entity-retrieval.mirror :as mirror]
[metabase.metabot.tools.entity-retrieval :as tools.entity-retrieval]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[next.jdbc :as jdbc]))
(set! *warn-on-reflection* true)
+;; the dedicated-harness tests below hit the app db before any auto-initializing mt helper — on the
+;; appdb-mode CI job this namespace can be an early db touch in a fresh JVM
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
(defn- approx [target] #(< (abs (- (double %) (double target))) 1e-9))
(deftest score-shape-matches-regular-search-test
@@ -34,7 +39,8 @@
(deftest dispatch-without-pgvector-test
(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"
;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:
- ;; available? is false, so search returns [] and the write-path nudge no-ops rather than throwing.
+ ;; entity retrieval is dedicated-only, so no URL means available? is false, search returns [] and the
+ ;; write-path nudge no-ops rather than throwing.
(mt/with-premium-features #{:library :library-retrieval}
(with-redefs [semantic.db.datasource/db-url nil]
(is (= [] (mirror/search "anything" 10)))
@@ -134,9 +140,11 @@
":library without :library-retrieval does not entitle the tool"))
(mt/with-premium-features #{:library :library-retrieval}
(is (true? (entity-retrieval.core/available?)))))
- ;; pgvector unconfigured -> unavailable regardless of license
+ ;; no URL -> unconfigured and unavailable regardless of license, even on a pgvector-capable
+ ;; Postgres app db: entity retrieval's tables aren't schema-isolated yet, so it stays dedicated-only
(with-redefs [semantic.db.datasource/db-url nil]
(mt/with-premium-features #{:library :library-retrieval}
+ (is (false? (entity-retrieval.core/pgvector-configured?)))
(is (false? (entity-retrieval.core/available?))))))
(testing "fully licensed + pgvector, but no way to compute embeddings -> unavailable"
(with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
index 4f94d1300c9..fbc949501b0 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
@@ -10,12 +10,17 @@
[metabase.measures.test-util :as measures.tu]
[metabase.metabot.tools.search :as tools.search]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as jdbc.rs]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
+;; raw t2/collections.tu access below runs before any auto-initializing mt helper, so the app db must be
+;; set up explicitly — on the appdb-mode CI job this namespace can be the first db touch in the JVM
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
;; OsiAiContext write hooks fire request-entity-sync!, which would spawn a background targeted reconcile
;; using the *real* configured embedding model and race these tests — they drive reconcile! / reconcile-entity!
;; with the mock model directly. Suppress the nudge here; the hook -> drain path is covered in core-test.
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
index 812f29e6ea0..cc4dd965337 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
@@ -3,7 +3,6 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
- [metabase.api.common :as api]
[metabase.metabot.tools.search :as search]
[metabase.permissions.core :as perms]
[metabase.search.core :as search-core]
@@ -27,82 +26,101 @@
`(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))
(deftest search-test
+ ;; Fully mocked: each sub-test redefs search-core/search, so no engine or index runs. active-engines is
+ ;; pinned to include semantic so search/search takes the two-engine fusion branch (term queries via the
+ ;; fallback engine, semantic queries via semantic), regardless of whether this instance has pgvector.
(mt/with-additional-premium-features #{:content-verification}
- (with-semantic-search-if-available!
- (mt/with-test-user :rasta
- (let [order-table {:id 1
- :model "table"
- :table_name "orders"
- :name "Orders"
- :description "Order table"
- :database_id 42
- :table_schema "public"}
- dashboard {:id 2
- :model "dashboard"
- :name "Sales Dashboard"
- :description "Dashboard for sales"
- :verified true}]
- (with-redefs [perms/impersonated-user? (fn [] false)
- perms/sandboxed-user? (fn [] false)
- api/*current-user-id* 1]
- (testing "search returns postprocessed results for term queries"
- (with-redefs [search-core/search (fn [_] {:data [order-table]})]
- (let [args {:term-queries ["orders"]
- :entity-types ["table"]}
- results (search/search args)
- expected [(#'search/postprocess-search-result order-table)]]
- (is (= expected results)))))
- (testing "search returns postprocessed results for semantic queries"
- (with-redefs [search-core/search (fn [_] {:data [dashboard]})]
- (let [args {:semantic-queries ["sales metrics"]
- :entity-types ["dashboard"]}
- results (search/search args)
- expected [(#'search/postprocess-search-result dashboard)]]
- (is (= expected results)))))
- (testing "search combines term and semantic queries using RRF"
- (with-redefs [search-core/search (fn [context]
- (if (= (:search-string context) "orders")
- {:data [order-table]}
- {:data [dashboard]}))]
- (let [args {:term-queries ["orders"]
- :semantic-queries ["sales"]
- :entity-types ["table" "dashboard"]}
- results (search/search args)]
- ;; Should return both results combined via RRF
- (is (= 2 (count results)))
- (is (some #(= (:id %) 1) results))
- (is (some #(= (:id %) 2) results)))))
- (testing "search applies RRF to overlapping results"
- (with-redefs [search-core/search (fn [_]
- {:data [order-table dashboard]})]
- (let [args {:term-queries ["orders" "sales"]
- :entity-types ["table" "dashboard"]}
- results (search/search args)]
- ;; Both queries return same results, RRF should boost them
- (is (= 2 (count results)))
- (is (some #(= (:id %) 1) results))
- (is (some #(= (:id %) 2) results)))))
- (testing "search handles empty results"
- (with-redefs [search-core/search (fn [_] {:data []})]
- (let [args {:term-queries ["nonexistent"]
- :entity-types ["table"]}
- results (search/search args)]
- (is (empty? results)))))
- (testing "search with metabot verified-or-curated content flag"
- (let [metabot {:entity_id "test-bot"
- :use_verified_content true}]
- (with-redefs [t2/select-one (fn [model & _]
- (is (= :model/Metabot model) "Should query for Metabot model")
- metabot)
- search-core/search (fn [context]
- ;; use_verified_content now drives the curated filter, not :verified
- (is (true? (:curated? context)))
- {:data [dashboard]})]
- (let [results (search/search {:term-queries ["test"]
- :metabot-id "test-bot"
- :entity-types ["dashboard"]})]
- (is (= 1 (count results)))
- (is (= 2 (:id (first results))))))))))))))
+ (mt/with-test-user :rasta
+ (let [order-table {:id 1
+ :model "table"
+ :table_name "orders"
+ :name "Orders"
+ :description "Order table"
+ :database_id 42
+ :table_schema "public"}
+ dashboard {:id 2
+ :model "dashboard"
+ :name "Sales Dashboard"
+ :description "Dashboard for sales"
+ :verified true}]
+ (with-redefs [perms/impersonated-user? (fn [] false)
+ perms/sandboxed-user? (fn [] false)
+ search.engine/active-engines (constantly [:search.engine/semantic :search.engine/appdb])]
+ (testing "search returns postprocessed results for term queries"
+ (with-redefs [search-core/search (fn [_] {:data [order-table]})]
+ (let [args {:term-queries ["orders"]
+ :entity-types ["table"]}
+ results (search/search args)
+ expected [(#'search/postprocess-search-result order-table)]]
+ (is (= expected results)))))
+ (testing "search returns postprocessed results for semantic queries"
+ (with-redefs [search-core/search (fn [_] {:data [dashboard]})]
+ (let [args {:semantic-queries ["sales metrics"]
+ :entity-types ["dashboard"]}
+ results (search/search args)
+ expected [(#'search/postprocess-search-result dashboard)]]
+ (is (= expected results)))))
+ (testing "search combines term and semantic queries using RRF"
+ (with-redefs [search-core/search (fn [context]
+ (if (= (:search-string context) "orders")
+ {:data [order-table]}
+ {:data [dashboard]}))]
+ (let [args {:term-queries ["orders"]
+ :semantic-queries ["sales"]
+ :entity-types ["table" "dashboard"]}
+ results (search/search args)]
+ ;; Should return both results combined via RRF
+ (is (= 2 (count results)))
+ (is (some #(= (:id %) 1) results))
+ (is (some #(= (:id %) 2) results)))))
+ (testing "with no semantic engine active, semantic queries fold into the default engine"
+ (with-redefs [search.engine/active-engines (constantly [:search.engine/appdb])
+ ;; Pass terms through uncombined: Postgres appdb would OR them into one string,
+ ;; which breaks the per-query mock keying below. Keeps this deterministic on any app-db.
+ search.engine/disjunction (fn [_ terms] terms)
+ search-core/search (fn [context]
+ (if (= (:search-string context) "orders")
+ {:data [order-table]}
+ {:data [dashboard]}))]
+ (let [args {:term-queries ["orders"]
+ :semantic-queries ["sales"]
+ :entity-types ["table" "dashboard"]}
+ results (search/search args)]
+ ;; The semantic query is not dropped — it is searched via the default engine and fused in.
+ (is (= 2 (count results)))
+ (is (some #(= (:id %) 1) results))
+ (is (some #(= (:id %) 2) results)))))
+ (testing "search applies RRF to overlapping results"
+ (with-redefs [search-core/search (fn [_]
+ {:data [order-table dashboard]})]
+ (let [args {:term-queries ["orders" "sales"]
+ :entity-types ["table" "dashboard"]}
+ results (search/search args)]
+ ;; Both queries return same results, RRF should boost them
+ (is (= 2 (count results)))
+ (is (some #(= (:id %) 1) results))
+ (is (some #(= (:id %) 2) results)))))
+ (testing "search handles empty results"
+ (with-redefs [search-core/search (fn [_] {:data []})]
+ (let [args {:term-queries ["nonexistent"]
+ :entity-types ["table"]}
+ results (search/search args)]
+ (is (empty? results)))))
+ (testing "search with metabot verified-or-curated content flag"
+ (let [metabot {:entity_id "test-bot"
+ :use_verified_content true}]
+ (with-redefs [t2/select-one (fn [model & _]
+ (is (= :model/Metabot model) "Should query for Metabot model")
+ metabot)
+ search-core/search (fn [context]
+ ;; use_verified_content now drives the curated filter, not :verified
+ (is (true? (:curated? context)))
+ {:data [dashboard]})]
+ (let [results (search/search {:term-queries ["test"]
+ :metabot-id "test-bot"
+ :entity-types ["dashboard"]})]
+ (is (= 1 (count results)))
+ (is (= 2 (:id (first results)))))))))))))
;; Mock embeddings: similar vectors for semantic synonym pairs, orthogonal vectors for unrelated terms.
(def ^:private test-mock-embeddings
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj
new file mode 100644
index 00000000000..9a81ec992f1
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj
@@ -0,0 +1,135 @@
+(ns metabase-enterprise.semantic-search.appdb-pgvector-mode-test
+ "End-to-end round trip for pgvector-on-the-app-db mode: no MB_PGVECTOR_DB_URL, semantic search runs
+ against the application database inside the semantic_search schema, sharing the app-db pool.
+
+ Opt-in and self-gated: the destructive round trip (installs the vector extension, drops the
+ semantic_search schema on cleanup) only runs with MB_APPDB_PGVECTOR_MODE_TEST=true — set by the
+ appdb-mode CI job, whose pgvector/pgvector app-db image it then REQUIRES (skips fail loudly there).
+ Deliberately does NOT use test-util's once-fixture — that fixture gates on the dedicated-harness
+ MB_PGVECTOR_DB_URL, which is exactly what this test runs without."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.app-db.core :as mdb]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
+
+(set! *warn-on-reflection* true)
+
+;; the mode probe deliberately answers only after the app db is set up (mdb/db-is-set-up?), and this
+;; namespace can be the first thing a fresh test JVM runs
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(deftest ^:parallel get-index-metadata-mode-test
+ (testing "get-index-metadata returns the schema-qualified config exactly in app-db mode"
+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/pgvector-mode (constantly :app-db)]
+ (is (= semantic.index-metadata/app-db-index-metadata (semantic.env/get-index-metadata))))
+ (doseq [mode [:dedicated :unavailable]]
+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/pgvector-mode (constantly mode)]
+ (is (= semantic.index-metadata/default-index-metadata (semantic.env/get-index-metadata)))))))
+
+(defn- appdb-can-host-pgvector?
+ "Real-environment gate for the round trip: a Postgres app db where the vector extension is installed or
+ installable. Read-only — activation (init-semantic-search!) installs the extension and creates the
+ schema."
+ []
+ (and (= :postgres (mdb/db-type))
+ (try
+ (semantic.db.datasource/check-app-db-pgvector-support)
+ (catch Exception _ false))))
+
+(defn- tables-in-schema
+ [connectable schema]
+ (->> (jdbc/execute! connectable
+ ["SELECT tablename FROM pg_tables WHERE schemaname = ?" schema]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (map :tablename)
+ set))
+
+(defn- schema-exists?
+ [connectable schema]
+ (boolean (seq (jdbc/execute! connectable
+ ["SELECT 1 FROM information_schema.schemata WHERE schema_name = ?" schema]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))
+
+(defn- opted-in?
+ "True when MB_APPDB_PGVECTOR_MODE_TEST=true: the appdb-mode CI job (or a developer deliberately opting
+ in). The round trip installs the vector extension and drops the semantic_search schema on cleanup, so it
+ never runs implicitly against whatever Postgres app db happens to be around."
+ []
+ (Boolean/parseBoolean (System/getenv "MB_APPDB_PGVECTOR_MODE_TEST")))
+
+(deftest ^:synchronized appdb-pgvector-round-trip-test
+ (cond
+ (not (opted-in?))
+ (testing "destructive round trip requires the MB_APPDB_PGVECTOR_MODE_TEST opt-in — skipping"
+ (is true))
+
+ (not (appdb-can-host-pgvector?))
+ (testing "opted in, but the app db can't host pgvector — the CI service must be misconfigured"
+ (is false "the appdb-mode job expects a pgvector-capable Postgres app db"))
+
+ ;; the app db already carries a semantic_search schema (a real app-db-mode deployment, or even an empty
+ ;; schema) — this test's cleanup DROP SCHEMA CASCADE would clobber it, so refuse rather than destroy it
+ (schema-exists? (mdb/data-source) "semantic_search")
+ (testing "opted in, but a semantic_search schema already exists — refusing to clobber it"
+ (is false "the appdb-mode job should start from a fresh app db with no semantic_search schema"))
+
+ :else
+ (mt/with-premium-features #{:semantic-search}
+ ;; the pair mirrors mock-embeddings' doc/query geometry: nearly identical vectors, so the query
+ ;; lands well inside the distance threshold for its document and nothing else
+ (semantic.tu/with-mock-embeddings {"Dog Training Guide" [0.12 -0.34 0.56 -0.78]
+ "puppy" [0.13 -0.33 0.57 -0.77]}
+ (with-redefs [semantic.db.datasource/db-url nil
+ semantic.db.datasource/data-source (atom nil)
+ ;; a cooldown left open by another test would make pgvector-mode read :unavailable here
+ semantic.db.datasource/app-db-pgvector-support (atom nil)
+ semantic.db.datasource/probe-cooldown-timer (atom nil)
+ semantic.db.datasource/logged-pgvector-absent? (atom false)
+ semantic.embedding/get-configured-model (fn [] semantic.tu/mock-embedding-model)]
+ (let [app-db (mdb/data-source)]
+ (try
+ (testing "activation is automatic: Postgres app db + extension, no URL"
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))
+ (is (semantic.db.datasource/pgvector-configured?)))
+ (testing "the module uses the shared app-db pool and the schema-qualified config"
+ (is (identical? app-db (semantic.env/get-pgvector-datasource!)))
+ (is (nil? @semantic.db.datasource/data-source)
+ "the shared pool must never land in the module's own pool atom")
+ (is (= "semantic_search" (:schema (semantic.env/get-index-metadata)))))
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ public-before (tables-in-schema app-db "public")]
+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata
+ (semantic.env/get-configured-embedding-model))
+ (semantic.pgvector-api/index-documents! pgvector index-metadata (semantic.tu/mock-documents))
+ (testing "every module table lives inside the semantic_search schema"
+ (let [tables (tables-in-schema app-db "semantic_search")]
+ (is (contains? tables "migration"))
+ (is (contains? tables "index_metadata"))
+ (is (contains? tables "index_control"))
+ (is (contains? tables "index_gate"))
+ (is (contains? tables "index_mock_model_4"))
+ (is (some #(str/starts-with? % "dlq_") tables))))
+ (testing "search round-trips through the app-db pool"
+ ;; results are reconstructed from legacy_input, so the mock card comes back as model+id
+ (is (= [{:model "card" :id 123}]
+ (->> (mt/with-test-user :crowberto
+ (semantic.pgvector-api/query pgvector index-metadata
+ {:search-string "puppy" :archived? false}))
+ :results
+ (filter (comp #{"card"} :model))
+ (mapv #(select-keys % [:model :id]))))))
+ (testing "the application schema is untouched"
+ (is (= public-before (tables-in-schema app-db "public")))))
+ (finally
+ (jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"])))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
index 534e04e5e58..f6c64514d7a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
@@ -2,9 +2,11 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as mdb]
[metabase.search.appdb.core :as appdb]
@@ -16,6 +18,18 @@
(use-fixtures :once #'semantic.tu/once-fixture)
+(deftest supported?-requires-an-embedder-test
+ (testing "a pgvector store alone is not enough to select the engine — auto-activation without a
+ configured embedding backend would loop every index and query embed into failure"
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-available? (constantly true)]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly {:provider "no-such-provider"})]
+ (is (false? (semantic.core/supported?))))
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly semantic.tu/mock-embedding-model)]
+ (is (true? (semantic.core/supported?))))))))
+
(deftest fallback-engine-available-with-semantic-test
(mt/with-premium-features #{:semantic-search}
(mt/with-temporary-setting-values [search.settings/search-engine "semantic"]
@@ -40,7 +54,10 @@
reset-called-atoms! #(do (reset! semantic-called? false)
(reset! appdb-called? false)
(reset! legacy-called? false))]
- (mt/with-dynamic-fn-redefs [semantic.pgvector-api/query
+ (mt/with-dynamic-fn-redefs [;; supported? requires a configured embedder; pin the mock so semantic is selectable
+ semantic.embedding/get-configured-model
+ (constantly semantic.tu/mock-embedding-model)
+ semantic.pgvector-api/query
(fn [& _]
(reset! semantic-called? true)
{:results [{:id 1 :name "semantic-result" :model "card" :collection_id 1 :score 0}]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj
new file mode 100644
index 00000000000..ef0b572bd0c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj
@@ -0,0 +1,200 @@
+(ns metabase-enterprise.semantic-search.db.datasource-mode-test
+ "Unit tests (redefs only, no databases) for pgvector mode selection: dedicated URL vs. shared app-db vs.
+ unavailable. The end-to-end app-db mode round-trip lives in
+ metabase-enterprise.semantic-search.appdb-pgvector-mode-test."
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.app-db.core :as mdb]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc])
+ (:import
+ (clojure.lang ExceptionInfo)))
+
+(set! *warn-on-reflection* true)
+
+(defmacro ^:private with-support-cache
+ "Run body with all three pieces of app-db probe state rebound to fresh atoms: the support cache (holding
+ `init`), the cooldown timer, and the log-once hint latch."
+ [init & body]
+ `(with-redefs [semantic.db.datasource/app-db-pgvector-support (atom ~init)
+ semantic.db.datasource/probe-cooldown-timer (atom nil)
+ semantic.db.datasource/logged-pgvector-absent? (atom false)]
+ ~@body))
+
+(deftest dedicated-mode-wins-test
+ (testing "MB_PGVECTOR_DB_URL always wins, even when the app db could also support pgvector"
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://localhost:5432/pgvector"
+ mdb/db-type (constantly :postgres)]
+ (with-support-cache true
+ (is (= :dedicated (semantic.db.datasource/pgvector-mode)))
+ (is (semantic.db.datasource/pgvector-configured?))))))
+
+(deftest whitespace-url-counts-as-unset-test
+ (testing "a whitespace-only MB_PGVECTOR_DB_URL is unset for every predicate — no silent app-db fallback
+ while a task gate elsewhere believes a dedicated store is configured"
+ (with-redefs [semantic.db.datasource/db-url " "
+ mdb/db-type (constantly :h2)]
+ (with-support-cache nil
+ (is (false? (semantic.db.datasource/dedicated-url-configured?)))
+ (is (false? (semantic.util/semantic-search-configured?)))
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))))))
+
+(deftest non-postgres-app-db-test
+ (testing "no URL + non-Postgres app db → :unavailable without ever probing the app db"
+ (doseq [db-type [:h2 :mysql]]
+ (with-redefs [semantic.db.datasource/db-url nil
+ mdb/db-type (constantly db-type)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not probe a non-Postgres app db")))]
+ (with-support-cache nil
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))
+ (is (not (semantic.db.datasource/pgvector-configured?)))
+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))))))
+
+(deftest support-requires-provisionable-store-test
+ (testing "supported only when the vector extension and the semantic_search schema exist or can be created"
+ (letfn [(check [] (semantic.db.datasource/check-app-db-pgvector-support))
+ (catalog [m] (fn [& _] m))]
+ (with-redefs [mdb/data-source (constantly ::app-pool)]
+ (testing "extension neither installed nor available → unsupported, no provisioning probe"
+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available false :schema-exists false})
+ semantic.db.datasource/app-db-can-provision-pgvector?
+ (fn [& _] (throw (AssertionError. "must not probe when the extension is unavailable")))]
+ (is (false? (check)))))
+ (testing "extension installed and schema present → supported without a DDL probe"
+ (with-redefs [jdbc/execute-one! (catalog {:installed true :available true :schema-exists true})
+ semantic.db.datasource/app-db-can-provision-pgvector?
+ (fn [& _] (throw (AssertionError. "must not probe when already fully provisioned")))]
+ (is (true? (check)))))
+ (testing "installed but schema missing → probe schema creation only, not the extension"
+ (with-redefs [jdbc/execute-one! (catalog {:installed true :available true :schema-exists false})
+ semantic.db.datasource/app-db-can-provision-pgvector?
+ (fn [_ create-extension? create-schema?]
+ (is (= [false true] [create-extension? create-schema?]))
+ true)]
+ (is (true? (check)))))
+ (testing "available but not installed → probe both extension and schema creation"
+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists false})
+ semantic.db.datasource/app-db-can-provision-pgvector?
+ (fn [_ create-extension? create-schema?]
+ (is (= [true true] [create-extension? create-schema?]))
+ true)]
+ (is (true? (check)))))
+ (testing "available but not installed, schema pre-created → probe the extension only"
+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists true})
+ semantic.db.datasource/app-db-can-provision-pgvector?
+ (fn [_ create-extension? create-schema?]
+ (is (= [true false] [create-extension? create-schema?]))
+ true)]
+ (is (true? (check)))))
+ (testing "a privilege gap while provisioning → unsupported"
+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists false})
+ semantic.db.datasource/app-db-can-provision-pgvector? (fn [& _] false)]
+ (is (false? (check)))))))))
+
+(deftest support-check-caching-test
+ (with-redefs [semantic.db.datasource/db-url nil
+ mdb/db-type (constantly :postgres)]
+ (testing "nothing cached (and mode :unavailable) before the app db is set up"
+ (with-support-cache nil
+ (with-redefs [mdb/db-is-set-up? (constantly false)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not probe before the app db is set up")))]
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))
+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))))
+ (testing "check runs exactly once; its result is cached for subsequent calls"
+ (with-support-cache nil
+ (let [calls (atom 0)]
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (fn [] (swap! calls inc) true)]
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))
+ (is (= 1 @calls))
+ (is (true? @semantic.db.datasource/app-db-pgvector-support))))))
+ (testing "a failed probe reads as unavailable, does NOT latch, and backs off before retrying"
+ (with-support-cache nil
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (fn [] (throw (ex-info "boom" {})))]
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))
+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))
+ (testing "within the backoff window the check must not run again"
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not re-probe during backoff")))]
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))))
+ (testing "once the window clears, the next call re-probes"
+ (reset! semantic.db.datasource/probe-cooldown-timer nil)
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))
+ ;; This exercises the false->true transition with the probe mocked.
+ ;; The real thing (an app-db role reads unsupported, an admin runs CREATE EXTENSION out-of-band, the
+ ;; next probe reads installed and the mode flips with no restart) was verified by hand against a real
+ ;; Postgres.
+ ;; We deliberately don't automate it: it needs a real DB whose extension state changes mid-test, which
+ ;; would be flaky against the database shared with other tests between runs.
+ (testing "an unsupported probe is NOT latched — it re-probes after the cooldown, so a runtime install is picked up"
+ (with-support-cache nil
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (constantly false)]
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))
+ (is (nil? @semantic.db.datasource/app-db-pgvector-support))
+ (testing "within the cooldown the check must not run again"
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not re-probe during cooldown")))]
+ (is (= :unavailable (semantic.db.datasource/pgvector-mode))))))
+ (testing "cooldown elapsed: pgvector installed at runtime is now picked up (no restart)"
+ (reset! semantic.db.datasource/probe-cooldown-timer nil)
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))
+ (testing "a confirmed true latches for the JVM lifetime; tests/REPL reset it"
+ (with-support-cache nil
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))
+ (is (true? @semantic.db.datasource/app-db-pgvector-support)))
+ (testing "latched: a now-failing check is never consulted"
+ (with-redefs [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not re-probe after a confirmed true")))]
+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))))
+
+(deftest unlicensed-availability-check-does-not-probe-test
+ (testing "without the :semantic-search feature, the availability gate never probes the app db"
+ (with-redefs [semantic.db.datasource/db-url nil
+ mdb/db-type (constantly :postgres)
+ mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/check-app-db-pgvector-support
+ (fn [] (throw (AssertionError. "must not probe when the feature is off")))]
+ (with-support-cache nil
+ (mt/with-premium-features #{}
+ (is (false? (semantic.util/semantic-search-available?))))
+ (is (nil? @semantic.db.datasource/app-db-pgvector-support))))))
+
+(deftest ensure-initialized-data-source-app-db-test
+ (testing "app-db mode hands back the shared application pool without storing it"
+ (with-redefs [semantic.db.datasource/db-url nil
+ semantic.db.datasource/data-source (atom nil)
+ mdb/db-type (constantly :postgres)
+ mdb/data-source (constantly ::app-pool)]
+ (with-support-cache true
+ (is (= ::app-pool (semantic.db.datasource/ensure-initialized-data-source!)))
+ (testing "the shared pool is never stored in the module's data-source atom"
+ (is (nil? @semantic.db.datasource/data-source)))
+ (testing "shutdown-db! therefore cannot close the shared pool"
+ ;; would throw on ::app-pool (not a PooledDataSource) if it tried
+ (semantic.db.datasource/shutdown-db!)
+ (is (= ::app-pool (semantic.db.datasource/ensure-initialized-data-source!))))))))
+
+(deftest ensure-initialized-data-source-unavailable-test
+ (testing "no pgvector anywhere → an actionable error"
+ (with-redefs [semantic.db.datasource/db-url nil
+ semantic.db.datasource/data-source (atom nil)
+ mdb/db-type (constantly :h2)]
+ (with-support-cache nil
+ (is (thrown-with-msg? ExceptionInfo #"MB_PGVECTOR_DB_URL"
+ (semantic.db.datasource/ensure-initialized-data-source!)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
index b3035593fbe..418aba5688d 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj
@@ -122,6 +122,38 @@
(is (= 2 (:retry_count (first results))))
(is (= t2 (:error_gated_at (first results))))))))))
+(deftest dlq-entry-upsert-schema-qualified-test
+ (testing "the conflict predicate references the target by bare name when the DLQ table is schema-qualified
+ (app-db mode); a qualified reference would not resolve inside ON CONFLICT DO UPDATE"
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ schema (str "dlq_test_schema_" (System/nanoTime))
+ index-metadata (assoc (semantic.tu/unique-index-metadata)
+ :schema schema
+ :index-table-qualifier (str schema ".%s"))
+ index-id 42]
+ (try
+ (jdbc/execute! pgvector [(str "CREATE SCHEMA " schema)])
+ (semantic.dlq/create-dlq-table-if-not-exists! pgvector index-metadata index-id)
+ (let [t1 (ts "2025-01-01T10:00:00Z")
+ t2 (ts "2025-01-01T11:00:00Z")
+ entry (fn [gated-at retry]
+ [{:gate_id "gate1"
+ :retry_count retry
+ :attempt_at t1
+ :last_attempted_at t2
+ :error_gated_at gated-at}])]
+ (is (= 1 (semantic.dlq/add-entries! pgvector index-metadata index-id (entry t1 0))))
+ (is (= 1 (semantic.dlq/add-entries! pgvector index-metadata index-id (entry t2 2)))
+ "the conflict-update path must work against a schema-qualified table")
+ (let [table-name (semantic.dlq/dlq-table-name-kw index-metadata index-id)
+ [row] (jdbc/execute! pgvector
+ (sql/format {:select [:*] :from [table-name]} :quoted true)
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (is (= 2 (:retry_count row)))
+ (is (= t2 (:error_gated_at row)))))
+ (finally
+ (jdbc/execute! pgvector [(str "DROP SCHEMA IF EXISTS " schema " CASCADE")]))))))
+
(deftest dlq-poll-test
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index 239f788011c..eb29a846d83 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -17,6 +17,7 @@
[metabase.llm.settings :as llm.settings]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.json :as json]
[toucan2.core :as t2])
(:import
@@ -25,6 +26,10 @@
(set! *warn-on-reflection* true)
+;; the token-tracking test hits the app db before any auto-initializing mt helper when it is the first
+;; db touch in a fresh JVM
+(use-fixtures :once (fixtures/initialize :db))
+
(deftest test-get-provider
(testing "get-active-model returns based on setting"
(mt/with-temporary-setting-values [ee-embedding-provider "ai-service"
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 1f57e677627..e52ec127174 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -19,7 +19,8 @@
[metabase.permissions.core :as perms]
[metabase.search.core :as search]
[metabase.test :as mt]
- [metabase.util :as u])
+ [metabase.util :as u]
+ [next.jdbc :as jdbc])
(:import
(org.postgresql.util PGobject)))
@@ -206,6 +207,20 @@
(is (zero? (semantic.tu/index-count index)))
(is (=? (expected-index-defs index) (semantic.tu/table-indexes table-name))))))))
+(deftest create-index-table!-extension-failure-test
+ (testing "a failed CREATE EXTENSION surfaces the actionable pgvector guidance, not the generic wrapper"
+ (mt/with-premium-features #{:semantic-search}
+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ ;; the extension is the first statement create! runs, so throwing here exercises the inner catch
+ (mt/with-dynamic-fn-redefs [jdbc/execute!
+ (fn [& _] (throw (RuntimeException. "permission denied to create extension")))]
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (semantic.index/create-index-table-if-not-exists!
+ (semantic.env/get-pgvector-datasource!) @index-ref)))]
+ (testing "the escaping error keeps the extension-install type and message, not \"Failed to create index table\""
+ (is (= ::semantic.index/extension-install-failed (:type (ex-data e))))
+ (is (re-find #"pgvector extension" (ex-message e))))))))))
+
(deftest create-hnsw-index-if-not-exists!-test
(mt/with-premium-features #{:semantic-search}
(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
index 554ae6906af..de40a2b0254 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
@@ -11,7 +11,9 @@
[metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.collections.models.collection :as collection]
[metabase.test :as mt]
[metabase.util :as u]
@@ -34,9 +36,9 @@
semantic.db.migration.impl/migrate-dynamic-schema! (constantly nil)]
(log.capture/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.db.migration :info]]
(semantic.db.connection/with-migrate-tx [tx]
- (semantic.db.migration/maybe-migrate! tx nil)
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata semantic.index-metadata/default-index-metadata})
{:messages (messages)
- :db-version (@#'semantic.db.migration/db-version tx)}))))]
+ :db-version (@#'semantic.db.migration/db-version semantic.index-metadata/default-index-metadata tx)}))))]
(testing "Migration up works"
(u/prog1 (migrate-and-get-db-version 130)
(is (= 130 (:db-version <>)))
@@ -56,6 +58,76 @@
(is (m/find-first (comp #{"Migration already performed, skipping."} :message)
(:messages <>)))))))))
+(deftest schema-scoped-migration-drop-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (testing "app-db-mode migration reset only ever drops tables inside the module's schema"
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata semantic.index-metadata/app-db-index-metadata
+ schema-tables (fn []
+ (->> (jdbc/execute! pgvector
+ ["SELECT tablename FROM pg_tables WHERE schemaname = 'semantic_search'"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (map :tablename)
+ set))]
+ ;; decoys playing the role of application tables; the second is a real app-db table name
+ ;; (migration 056) squarely inside the module's naming family — the worst case for any
+ ;; name-pattern-based scoping
+ (jdbc/execute! pgvector ["CREATE TABLE decoy_app_table (id int)"])
+ (jdbc/execute! pgvector ["CREATE TABLE semantic_search_token_tracking (id int)"])
+ ;; positive control: a leftover table inside the module schema is fair game for the reset
+ (jdbc/execute! pgvector ["CREATE SCHEMA semantic_search"])
+ (jdbc/execute! pgvector ["CREATE TABLE semantic_search.legacy_junk (id int)"])
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))
+ (testing "tables outside the schema survive the version<2 drop-everything reset"
+ (is (semantic.util/table-exists? pgvector "public.decoy_app_table"))
+ (is (semantic.util/table-exists? pgvector "public.semantic_search_token_tracking")))
+ (testing "inside the schema: junk dropped, module tables created"
+ (is (= #{"migration" "index_metadata" "index_control" "index_gate"}
+ (schema-tables)))))))))
+
+(deftest dedicated-reset-stays-in-default-schema-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (testing "the dedicated-mode reset wipes only the default schema — cohabitant schemas survive"
+ (let [pgvector (semantic.env/get-pgvector-datasource!)]
+ (jdbc/execute! pgvector ["CREATE SCHEMA cohabitant"])
+ (jdbc/execute! pgvector ["CREATE TABLE cohabitant.precious (id int)"])
+ (jdbc/execute! pgvector ["CREATE TABLE doomed (id int)"])
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata semantic.index-metadata/default-index-metadata}))
+ (is (true? (semantic.util/table-exists? pgvector "cohabitant.precious")))
+ (is (false? (semantic.util/table-exists? pgvector "public.doomed"))))))))
+
+(deftest dedicated-reset-refuses-app-db-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (testing "the dedicated-mode reset refuses a database that looks like a Metabase app db"
+ (let [pgvector (semantic.env/get-pgvector-datasource!)]
+ ;; core_user is the app-db sentinel; a generic Liquibase table alongside it must not mask the refusal
+ (jdbc/execute! pgvector ["CREATE TABLE databasechangelog (id int)"])
+ (jdbc/execute! pgvector ["CREATE TABLE core_user (id int)"])
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Metabase application"
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate!
+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))))
+ (testing "nothing was dropped"
+ (is (true? (semantic.util/table-exists? pgvector "public.core_user")))))))))
+
+(deftest dedicated-reset-allows-liquibase-only-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (testing "a generic Liquibase databasechangelog does not, on its own, look like a Metabase app db"
+ (let [pgvector (semantic.env/get-pgvector-datasource!)]
+ (jdbc/execute! pgvector ["CREATE TABLE databasechangelog (id int)"])
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate!
+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))
+ (testing "migration proceeds: the module tables are created and the stray table is wiped"
+ (is (true? (semantic.util/table-exists? pgvector "public.index_metadata")))
+ (is (false? (semantic.util/table-exists? pgvector "public.databasechangelog")))))))))
+
(defn- executions-overlap?
"Check if any two executions overlap in time. Each entry is [tid :started/:ended timestamp].
Returns true if there's any overlap (which would indicate lock failure)."
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
index f9b62da33a6..4918fa67a81 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
@@ -19,6 +19,17 @@
(set! *warn-on-reflection* true)
+(deftest ^:parallel schema-scoping-helpers-test
+ (testing "scope-where-to-schema adds the schema predicate only in shared app-db mode (non-nil schema)"
+ (is (= [:and [:like :t.table_name [:inline "x_%"]]]
+ (#'sut/scope-where-to-schema [:and [:like :t.table_name [:inline "x_%"]]] nil)))
+ (is (= [:and [:like :t.table_name [:inline "x_%"]] [:= :t.table_schema [:inline "semantic_search"]]]
+ (#'sut/scope-where-to-schema [:and [:like :t.table_name [:inline "x_%"]]] "semantic_search"))))
+ (testing "requalify-table-names prefixes the schema only in shared app-db mode"
+ (is (= ["a" "b"] (#'sut/requalify-table-names nil ["a" "b"])))
+ (is (= ["semantic_search.a" "semantic_search.b"]
+ (#'sut/requalify-table-names "semantic_search" ["a" "b"])))))
+
(use-fixtures :once #'semantic.tu/once-fixture)
(defn- create-test-index
@@ -228,25 +239,25 @@
recent-time (t/minus (t/instant) (t/hours 1))] ; 1 hour ago
(testing "parse-repair-table-timestamp"
(let [repair-table-name (with-redefs [t/instant (constantly old-time)]
- (#'repair/repair-table-name))
+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))
parsed-time (#'sut/parse-repair-table-timestamp repair-table-name)]
(is (= (t/truncate-to old-time :millis)
parsed-time))))
(testing "orphan repair table detection and cleanup"
(let [old-repair-table-name (with-redefs [t/instant (constantly old-time)]
- (#'repair/repair-table-name))
+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))
recent-repair-table-name (with-redefs [t/instant (constantly recent-time)]
- (#'repair/repair-table-name))
+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))
non-repair-table-name "regular_table_123"]
(try
(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" old-repair-table-name)])
(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" recent-repair-table-name)])
(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" non-repair-table-name)])
(mt/with-dynamic-fn-redefs [semantic.settings/repair-table-retention-hours (constantly retention-hours)]
- (let [orphan-tables (#'sut/orphan-repair-tables pgvector)]
+ (let [orphan-tables (#'sut/orphan-repair-tables pgvector semantic.index-metadata/default-index-metadata)]
(is (= #{old-repair-table-name} (set orphan-tables))
"Only old repair table should be detected as orphan"))
- (#'sut/cleanup-orphan-repair-tables! pgvector)
+ (#'sut/cleanup-orphan-repair-tables! pgvector semantic.index-metadata/default-index-metadata)
(is (not (semantic.tu/table-exists-in-db? old-repair-table-name))
"Old repair table should be dropped")
(is (semantic.tu/table-exists-in-db? recent-repair-table-name)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
index b2b63a40733..f416db3c3bf 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
@@ -4,12 +4,14 @@
[honey.sql :as sql]
[java-time.api :as t]
[metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.task.metric-collector :as semantic.task.collector]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.test :as mt]
[next.jdbc :as jdbc]))
@@ -71,7 +73,12 @@
index-metadata (semantic.tu/unique-index-metadata)
model semantic.tu/mock-embedding-model]
(mt/with-dynamic-fn-redefs [semantic.env/get-index-metadata (fn [] index-metadata)
- semantic.env/get-configured-embedding-model (fn [] model)]
+ semantic.env/get-configured-embedding-model (fn [] model)
+ ;; supported? requires a configured embedder for engine selection
+ semantic.embedding/get-configured-model (fn [] model)
+ ;; collect-metrics! only runs when semantic is the active engine; pin it so a
+ ;; sibling test leaking the search-engine setting can't make this read 0
+ semantic.u/semantic-search-active? (fn [] true)]
(testing "Missing tables are handled gracefully"
(let [result (try
(@#'semantic.task.collector/collect-metrics!)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
index b2cf2fa2150..c595529ccd7 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer]
[metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.test.fixtures :as fixtures]
[toucan2.core :as t2])
(:import
(java.sql Timestamp)
@@ -10,6 +11,10 @@
(set! *warn-on-reflection* true)
+;; raw t2 access below runs before any auto-initializing mt helper — on the appdb-mode CI job this
+;; namespace can be an early db touch in a fresh JVM
+(use-fixtures :once (fixtures/initialize :db))
+
(deftest trim-test
(when (semantic.u/semantic-search-available?)
(t2/delete! :model/SemanticSearchTokenTracking)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index 27061cf11ae..f5e8629e22c 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -52,6 +52,11 @@
(defn do-with-temp-datasource!
"Impl [[with-temp-datasource]]."
[db-name thunk]
+ ;; with no URL the redef'd nil db-url could resolve to :app-db mode and hand back the application
+ ;; pool — the dedicated harness (which DROPs/CREATEs databases) must never point at the app db.
+ ;; unconditional (not assert): this guards destructive setup, and asserts can be elided
+ (when (str/blank? (:mb-pgvector-db-url env))
+ (throw (ex-info "with-temp-datasource! requires the dedicated-harness MB_PGVECTOR_DB_URL" {})))
(with-redefs [semantic.db.datasource/db-url (alt-db-name-url (:mb-pgvector-db-url env) db-name)
semantic.db.datasource/data-source (atom nil)]
(try
@@ -559,7 +564,7 @@
(semantic.index/drop-index-table! pgvector {:table-name table_name})
(semantic.dlq/drop-dlq-table-if-exists! pgvector index-metadata index-id))
(semantic.index-metadata/drop-tables-if-exists! pgvector index-metadata)
- (semantic.db.migration/drop-migration-table! pgvector))
+ (semantic.db.migration/drop-migration-table! index-metadata pgvector))
(defn get-table-names [pgvector]
(->> ["select table_name from information_schema.tables"]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
new file mode 100644
index 00000000000..9cff3b055ee
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
@@ -0,0 +1,50 @@
+(ns metabase-enterprise.semantic-search.util-test
+ (:require
+ [clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once #'semantic.tu/once-fixture)
+
+(deftest ^:parallel quote-ident-test
+ (testing "wraps in double quotes and doubles any embedded double quote (the next.jdbc.quoted/postgres contract)"
+ (is (= "\"semantic_search\"" (semantic.util/quote-ident "semantic_search")))
+ (is (= "\"weird\"\"name\"" (semantic.util/quote-ident "weird\"name")))))
+
+(deftest ^:parallel column-keyword-test
+ (testing "a dotted-name keyword (renders as separate identifiers), never a namespaced one"
+ (is (= :index_table_1.model (semantic.util/column-keyword "index_table_1" "model")))
+ (is (= :semantic_search.index_table_1.model
+ (semantic.util/column-keyword "semantic_search.index_table_1" "model")))
+ (is (nil? (namespace (semantic.util/column-keyword "semantic_search.t" "c")))))
+ (testing "conflict-target-column drops the schema — ON CONFLICT names the target by its bare relation"
+ (is (= :index_table_1.model
+ (semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))
+
+(deftest catalog-lookups-schema-scoping-test
+ (testing "qualified names scope table/index existence checks to their schema; a same-named object in
+ another schema (e.g. the app db's public) must not satisfy them"
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (let [pgvector (semantic.env/get-pgvector-datasource!)]
+ (jdbc/execute! pgvector ["CREATE TABLE decoy (id int)"])
+ (jdbc/execute! pgvector ["CREATE INDEX decoy_idx ON decoy (id)"])
+ (jdbc/execute! pgvector ["CREATE SCHEMA s2"])
+ (testing "unqualified names keep the historical any-schema behavior"
+ (is (true? (semantic.util/table-exists? pgvector "decoy")))
+ (is (true? (semantic.util/index-exists? pgvector "decoy_idx"))))
+ (testing "qualified names are not fooled by the public decoys"
+ (is (true? (semantic.util/table-exists? pgvector "public.decoy")))
+ (is (true? (semantic.util/index-exists? pgvector "public.decoy_idx")))
+ (is (false? (semantic.util/table-exists? pgvector "s2.decoy")))
+ (is (false? (semantic.util/index-exists? pgvector "s2.decoy_idx"))))
+ (testing "qualified names find the objects in their own schema"
+ (jdbc/execute! pgvector ["CREATE TABLE s2.decoy (id int)"])
+ (jdbc/execute! pgvector ["CREATE INDEX decoy_idx ON s2.decoy (id)"])
+ (is (true? (semantic.util/table-exists? pgvector "s2.decoy")))
+ (is (true? (semantic.util/index-exists? pgvector "s2.decoy_idx")))))))))