The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Support pgvector on the app DB with minimal config (#77371)
Chris Truter8ef62d8612b4
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.ymlindex dc6faf705dc..f20490a8ec6 100644--- a/.github/workflows/semantic-search.yml+++ b/.github/workflows/semantic-search.yml@@ -301,12 +301,79 @@ jobs:previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}+ # pgvector-on-the-app-db mode: the app db itself is a pgvector-capable Postgres and+ # MB_PGVECTOR_DB_URL is deliberately unset. The dedicated-harness tests gate themselves off; the+ # appdb-pgvector-mode round trip (and the URL-independent unit tests) run for real.+ semantic-search-tests-appdb-mode:+ if: ${{ !inputs.skip }}+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 40+ strategy:+ fail-fast: false+ matrix:+ version:+ - name: pgvector pg17 as app db, no dedicated pgvector+ junit-name: semantic-search-tests-appdb-mode-pg17+ appdb-image: pgvector/pgvector:pg17+ job:+ - name: Semantic Search Tests+ build-static-viz: false+ test-args: >-+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/entity_retrieval" "enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj"]'+ exclude-tag: ':mb/driver-tests'++ services:+ postgres:+ image: ${{ matrix.version.appdb-image }}+ ports:+ - "5432:5432"+ env:+ POSTGRES_USER: mb_test+ POSTGRES_DB: mb_test+ POSTGRES_HOST_AUTH_METHOD: trust+ env:+ CI: 'true'+ MB_DB_TYPE: postgres+ MB_DB_PORT: 5432+ MB_DB_HOST: localhost+ MB_DB_DBNAME: mb_test+ MB_DB_USER: mb_test+ # opts this job into the destructive appdb-pgvector-mode round trip, which then FAILS (rather+ # than silently skipping) if the app db can't actually host pgvector+ MB_APPDB_PGVECTOR_MODE_TEST: 'true'+ name: "${{ matrix.version.name }} ${{ matrix.job.name }}"+ steps:+ - uses: actions/checkout@v6+ - name: Test ${{ matrix.version.name }}+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ build-static-viz: ${{ matrix.job.build-static-viz }}+ junit-name: ${{ matrix.version.junit-name }}+ test-args: >-+ ${{ matrix.job.test-args }}+ :exclude-tags '[${{ matrix.job.exclude-tag }}]'+ quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}+ - name: Upload Test Results+ uses: ./.github/actions/upload-test-results+ if: always()+ with:+ input-path: ./target/junit/+ output-name: ${{ github.job }}+ bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}+ role-to-assume: ${{ secrets.CI_TEST_RESULTS_IAM_ROLE }}+ aws-region: ${{ vars.AWS_REGION }}+ trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}+ previous-step-outcome: ${{ steps.test-driver.outputs.test-outcome }}+ run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}+semantic-search-tests-result:needs:- semantic-search-tests-h2- semantic-search-tests-mariadb- semantic-search-tests-mysql- semantic-search-tests-postgres+ - semantic-search-tests-appdb-moderuns-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 5name: semantic-search-tests-resultdiff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.cljindex 086872b2b5c..f18db30057a 100644--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj@@ -30,11 +30,16 @@(jobs/key "metabase-enterprise.entity-retrieval.sync.job"))(defn pgvector-configured?- "True when a pgvector store is configured (read once at boot from MB_PGVECTOR_DB_URL).+ "True when a dedicated pgvector store is configured (MB_PGVECTOR_DB_URL).+ Deliberately excludes pgvector-on-the-app-db: entity retrieval's tables are not yet schema-isolated+ (bare `library_entity_index*` names, dropped and recreated on rebuild), so it stays dedicated-only+ until they are.The sync task gates scheduling on this rather than [[available?]], so the periodic safety net exists- even when the library-retrieval feature is turned on after startup (a common onboarding flow)."+ even when the library-retrieval feature is turned on after startup (a common onboarding flow).+ A plain URL check, not [[metabase-enterprise.semantic-search.db.datasource/pgvector-mode]]: mode+ resolution can probe the app db, which a dedicated-only scheduling gate has no business doing."[]- (string? (not-empty semantic.db.datasource/db-url)))+ (semantic.db.datasource/dedicated-url-configured?))(defn available?"Whether the entity-retrieval mirror can run right now. All four must hold:@@ -48,9 +53,11 @@The feature and config checks can flip at runtime (token/settings entered post-boot), so callersre-evaluate per use."[]- (and (pgvector-configured?)- (premium-features/has-feature? :library)+ ;; entitlement first: short-circuit on the license flags for instances that can't use the answer, before+ ;; the config check (here a cheap dedicated-URL string check, not the app-db pgvector probe)+ (and (premium-features/has-feature? :library)(premium-features/has-feature? :library-retrieval)+ (pgvector-configured?)(embedding/embedding-supported? (embedding/get-configured-model))))(defn- index-ready?diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/api.cljindex 127137972af..1a08198e44a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/api.clj@@ -2,6 +2,7 @@"/api/ee/semantic-search endpoints"(:require[clojure.core.memoize :as memoize]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]@@ -30,20 +31,22 @@:indexed_count <number of indexed items>:total_est <estimated total number of items to index>- If no index is active, returns an empty map."+ If no index is active, or no pgvector database is configured, returns an empty map."[](perms/check-has-application-permission :setting)- (let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)- active-index (when (and pgvector index-metadata)- (semantic.index-metadata/get-active-index-state pgvector index-metadata))]- (try- (if active-index- {:indexed_count (active-index-document-count pgvector active-index)- :total_est (indexible-items-count)}- {})- (catch Exception e- (throw (ex-info "Error fetching semantic search index status" {} e))))))+ (if-not (semantic.db.datasource/pgvector-configured?)+ {}+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)+ active-index (when (and pgvector index-metadata)+ (semantic.index-metadata/get-active-index-state pgvector index-metadata))]+ (try+ (if active-index+ {:indexed_count (active-index-document-count pgvector active-index)+ :total_est (indexible-items-count)}+ {})+ (catch Exception e+ (throw (ex-info "Error fetching semantic search index status" {} e)))))))(def ^{:arglists '([request respond raise])} routes"`/api/ee/semantic-search` routes."diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 71d065f447e..ca49170a47d 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -4,7 +4,7 @@[clojure.string :as str][medley.core :as m][metabase-enterprise.semantic-search.embedders]- [metabase-enterprise.semantic-search.embedding]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]@@ -20,6 +20,8 @@[potemkin :as p][toucan2.realize :as t2.realize]))+;; import-vars requires full namespace symbols, so it can't use the alias+#_{:clj-kondo/ignore [:aliased-namespace-symbol]}(p/import-vars[metabase-enterprise.semantic-search.embeddersactive-embedding-model@@ -37,7 +39,12 @@"Enterprise implementation of semantic search engine support check.":feature :semantic-search[]- (semantic.util/semantic-search-available?))+ ;; Gate engine selection on a usable embedder. App-db mode gives every Postgres instance a pgvector+ ;; store, so without this the engine auto-activates with no embedder configured and every index and+ ;; query embed fails. available?/capable? skip the gate on purpose: an existing index still needs+ ;; maintenance while the embedder is temporarily unconfigured.+ (and (semantic.util/semantic-search-available?)+ (semantic.embedding/embedding-supported? (semantic.embedding/get-configured-model))))(defn build-hnsw-index-async!"Build the HNSW index on the active semantic search index in the background, returning promptly.@@ -190,6 +197,7 @@(init! searchable-documents {}))(semantic.repair/with-repair-table!pgvector+ index-metadata(fn [repair-table-name];; Re-gate all provided documents, populating the repair table as we go(semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documentsdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.cljindex b299d7de5bc..a816698d2d6 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj@@ -2,10 +2,13 @@(:require[clojure.string :as str][environ.core :refer [env]]+ [metabase.app-db.core :as mdb][metabase.connection-pool :as connection-pool][metabase.util :as u][metabase.util.log :as log]- [next.jdbc :as jdbc])+ [next.jdbc :as jdbc]+ [next.jdbc.quoted :as quoted]+ [next.jdbc.result-set :as jdbc.rs])(:import(com.mchange.v2.c3p0 DataSources PooledDataSource)(java.net URLDecoder)@@ -15,13 +18,30 @@(set! *warn-on-reflection* true)(def data-source- "Atom to hold the pooled JDBC data source for the semantic search database."+ "Atom to hold the pooled JDBC data source for the semantic search database.+ Only ever holds a dedicated (MB_PGVECTOR_DB_URL) pool; in app-db mode the shared application pool is+ returned by [[ensure-initialized-data-source!]] without being stored here, so [[shutdown-db!]] can never+ close it."(atom nil))(def db-url"The database URL used to connect to pgvector"(env :mb-pgvector-db-url))+(defn dedicated-url-configured?+ "True when MB_PGVECTOR_DB_URL is set to a non-blank value.+ Canonical check for \"a dedicated pgvector store is configured\"; using it everywhere stops a+ whitespace-only URL from reading as configured in one subsystem and unset in another."+ []+ (not (str/blank? db-url)))++(def app-db-schema+ "The Postgres schema holding all semantic-search tables when sharing the application database.+ Isolating by schema makes destructive maintenance (see+ [[metabase-enterprise.semantic-search.db.migration.impl]]) structurally incapable of touching+ application tables."+ "semantic_search")+;; All pgvector config — connection *and* connection-pool parameters — rides MB_PGVECTOR_DB_URL, so a;; deployment sets everything in one place (the DB secret) with no env-var-vs-URL precedence to reason;; about. c3p0 pool params can't ride the JDBC URL into the driver, so we parse them out here (see@@ -177,7 +197,151 @@(init-db!)(test-connection!))+(def app-db-pgvector-support+ "Latches `true` once the app db is confirmed to support pgvector; nil until then.+ A confirmed extension doesn't vanish under a running instance, so `true` is permanent. A negative result+ is NOT latched — it opens a [[probe-cooldown-timer]] and is re-probed, so installing pgvector on a+ running instance is picked up without a restart. Tests reset it."+ (atom nil))++(def probe-cooldown-timer+ "A [[u/start-timer]] taken when the last app-db pgvector probe came back unsupported or errored, or nil+ when a probe is due. Bounds re-probing to once per [[probe-cooldown-ms]] so an absent extension doesn't+ re-query the catalog on every search and 20s indexer tick. Tests reset it."+ (atom nil))++(def ^:private probe-cooldown-ms+ "How long an unsupported or failed app-db pgvector probe is trusted before re-probing.+ Long enough that a never-provisioned instance isn't running a rolled-back CREATE probe into its DDL audit+ log every few seconds, short enough to pick up a runtime `CREATE EXTENSION` / privilege grant without a+ restart."+ (.toMillis (java.time.Duration/ofMinutes 5)))++(defonce ^{:doc "Log-once latch for the \"no pgvector\" operator hint; the negative probe recurs each+ cooldown, so without it the hint would repeat. Tests reset it."}+ logged-pgvector-absent?+ (atom false))++(defn- probe-due?+ "True when no app-db pgvector probe cooldown is active."+ []+ (let [timer @probe-cooldown-timer]+ (or (nil? timer) (>= (u/since-ms timer) probe-cooldown-ms))))++(defn- app-db-can-provision-pgvector?+ "Whether the app-db user can create whichever store pieces are still missing, checked without persisting+ them: the CREATEs run in a transaction that always rolls back.+ Attempts CREATE EXTENSION only when `create-extension?` and CREATE SCHEMA only when `create-schema?`, so+ an already-installed extension or existing schema needs no create privilege.+ A privilege error reads as false."+ [app-datasource create-extension? create-schema?]+ (try+ (jdbc/with-transaction [tx app-datasource {:rollback-only true}]+ (when create-extension?+ (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]))+ (when create-schema?+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))])))+ true+ (catch Exception e+ (log/debug e "Semantic search: the application database user cannot provision the pgvector store")+ false)))++(defn check-app-db-pgvector-support+ "Can the application database act as the pgvector store?+ True when the `vector` extension and the [[app-db-schema]] schema are present, or the app-db user can+ create whichever is missing.+ Provisioning is verified in a rolled-back transaction (CREATE EXTENSION / CREATE SCHEMA), not read from+ pg_available_extensions: managed Postgres often lists the extension as available while denying the DDL.+ The probe persists nothing, so the unlicensed and disabled instances whose availability predicates reach+ here never mutate the app db; the persisted CREATE EXTENSION / CREATE SCHEMA run only on the activation+ path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]])."+ []+ (let [app-datasource (mdb/data-source)+ ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not+ ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog+ ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to+ ;; unavailable rather than passing here and crashing later when init creates tables it can't write.+ {:keys [installed available schema-exists]}+ (jdbc/execute-one! app-datasource+ [(str "SELECT"+ " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"+ " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"+ " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")+ app-db-schema]+ {:builder-fn jdbc.rs/as-unqualified-kebab-maps})]+ (cond+ (not (or installed available)) false+ (and installed schema-exists) true+ :else (app-db-can-provision-pgvector? app-datasource+ (not installed)+ (not schema-exists)))))++(defn- app-db-pgvector-supported?+ "Whether the application database can act as the pgvector store, via a cached probe.+ A confirmed `true` latches for the JVM lifetime (see [[app-db-pgvector-support]]); an unsupported or+ errored probe is trusted only for [[probe-cooldown-ms]] before re-probing, so a runtime `CREATE+ EXTENSION` / package install is picked up without a restart while a persistent negative doesn't re-query+ and re-warn on every call. Returns false while the app db is not yet set up."+ []+ (if (true? @app-db-pgvector-support)+ true+ (boolean+ (when (and (mdb/db-is-set-up?) (probe-due?))+ (locking app-db-pgvector-support+ (cond+ (true? @app-db-pgvector-support) true+ ;; a racing thread probed first and opened the cooldown+ (not (probe-due?)) false+ :else+ (try+ (if (check-app-db-pgvector-support)+ (do+ (log/info (str "Semantic search: using the application database as the pgvector store"+ " (MB_PGVECTOR_DB_URL is not set). Set it if this instance should use a"+ " dedicated pgvector database."))+ (reset! app-db-pgvector-support true)+ (reset! probe-cooldown-timer nil)+ true)+ (do+ (when (compare-and-set! logged-pgvector-absent? false true)+ (log/info (str "Semantic search: the application database cannot host pgvector (the"+ " app-db user cannot create the vector extension or the semantic_search"+ " schema). Install pgvector and grant CREATE, or set MB_PGVECTOR_DB_URL;"+ " it is picked up automatically, no restart needed.")))+ (reset! probe-cooldown-timer (u/start-timer))+ false))+ (catch Exception e+ (reset! probe-cooldown-timer (u/start-timer))+ (log/warn e (str "Semantic search: pgvector support check on the application database failed;"+ " will retry after the cooldown."))+ false))))))))++(defn pgvector-mode+ "How this instance reaches its pgvector database:+ :dedicated MB_PGVECTOR_DB_URL is set (always wins)+ :app-db no URL, but the Postgres application database supports the vector extension+ :unavailable no pgvector anywhere — semantic search cannot run."+ []+ (cond+ (dedicated-url-configured?) :dedicated+ (and (= :postgres (mdb/db-type))+ (app-db-pgvector-supported?)) :app-db+ :else :unavailable))++(defn pgvector-configured?+ "Canonical availability predicate: does this instance have a pgvector database to work with?"+ []+ (not= :unavailable (pgvector-mode)))+(defn ensure-initialized-data-source!- "Return datasource. Initialize if necessary."+ "Return datasource. Initialize if necessary.+ In app-db mode this is the application database's own shared pool."[]- (or @data-source (init-db!)))+ (or @data-source+ (case (pgvector-mode)+ :dedicated (init-db!)+ :app-db (mdb/data-source)+ :unavailable (throw (ex-info (str "Semantic search requires a pgvector database: set MB_PGVECTOR_DB_URL,"+ " or use a Postgres application database with the pgvector extension"+ " available.")+ {:mode :unavailable})))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.cljindex 5a8769b04ac..f7f91748932 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj@@ -3,6 +3,7 @@[honey.sql :as sql][honey.sql.helpers :as sql.helpers][metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.util.log :as log][next.jdbc :as jdbc])(:import@@ -14,17 +15,39 @@[:migrated_at :timestamp [:default [:NOW]]][:status [:varchar 32]]])-(def ^:private migration-table-hsql- (-> (sql.helpers/create-table :migration :if-not-exists)- (sql.helpers/with-columns columns)))+(defn- migration-table-kw+ "The migration bookkeeping table: inside the module's schema when index-metadata carries one (shared+ app-db mode), bare `migration` otherwise."+ [index-metadata]+ (if-let [schema (:schema index-metadata)]+ (keyword (str schema ".migration"))+ :migration))++(defn- ensure-schema-exists!+ "Create the module's schema when index-metadata carries one.+ This is the first *persisted* write against the app db in app-db mode; insufficient privileges normally+ surface earlier, at the capability probe's rolled-back CREATE SCHEMA."+ [index-metadata tx]+ (when-let [schema (:schema index-metadata)]+ (try+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (semantic.util/quote-ident schema))])+ (catch SQLException e+ (throw (ex-info (format (str "Failed to create the %s schema on the application database. Grant"+ " CREATE on the database to the Metabase user, or set"+ " MB_PGVECTOR_DB_URL to use a dedicated pgvector database.")+ schema)+ {:type ::schema-creation-failed :schema schema}+ e))))))(defn- migration-table-sql- []- (sql/format migration-table-hsql))+ [index-metadata]+ (-> (sql.helpers/create-table (migration-table-kw index-metadata) :if-not-exists)+ (sql.helpers/with-columns columns)+ (sql/format)))(defn- ensure-migration-table!- [tx]- (let [sql (migration-table-sql)]+ [index-metadata tx]+ (let [sql (migration-table-sql index-metadata)](try(jdbc/execute! tx sql)(catch SQLException e@@ -35,23 +58,23 @@(defn- db-version"Get database version of schema from migration table."- [tx]+ [index-metadata tx](or (:max_version(jdbc/execute-one! tx(sql/format {:select [[[:max :version] :max_version]]- :from [:migration]})))+ :from [(migration-table-kw index-metadata)]})))-1))(defn- write-successful-migration!- [tx]- (jdbc/execute-one! tx (sql/format (-> (sql.helpers/insert-into :migration)+ [index-metadata tx]+ (jdbc/execute-one! tx (sql/format (-> (sql.helpers/insert-into (migration-table-kw index-metadata))(sql.helpers/values [{:version semantic.db.migration.impl/schema-version:status "success"}])))))(defn maybe-migrate-schema!"Execute schema migration (control, meta, gate, ...) if appropriate."- [tx opts]- (let [db-version (db-version tx)]+ [tx {:keys [index-metadata] :as opts}]+ (let [db-version (db-version index-metadata tx)](cond(= db-version semantic.db.migration.impl/schema-version)(log/info "Migration already performed, skipping.")@@ -61,7 +84,7 @@(log/infof "Starting migration from version %d to %d."db-version semantic.db.migration.impl/schema-version)(semantic.db.migration.impl/migrate-schema! tx opts)- (write-successful-migration! tx))+ (write-successful-migration! index-metadata tx)):else(log/infof "Database schema version (%d) is newer than code version (%d). Not performing migration."@@ -69,27 +92,23 @@nil)(defn- index-metadata-table-exists?- [tx]- (jdbc/execute-one! tx- (sql/format {:select [[[:raw 1] :exists]]- :from [:information_schema.tables]- :where [[:= :information_schema.tables.table_name [:inline "index_metadata"]]]- :limit 1})))+ [index-metadata tx]+ (semantic.util/table-exists? tx (:metadata-table-name index-metadata)))(defn- lowest-dynamic-db-version"Lowest index version. If there is lower than defined in code dynamic schema migration will be attempted."- [tx]- (or (when (index-metadata-table-exists? tx)+ [index-metadata tx]+ (or (when (index-metadata-table-exists? index-metadata tx)(:min_index(jdbc/execute-one! tx(sql/format {:select [[[:min :index_version] :min_index]]- :from [:index_metadata]}))))+ :from [(keyword (:metadata-table-name index-metadata))]}))))0))(defn maybe-migrate-dynamic-schema!"Migration for dynamic tables (index_table_xyzs) if appropriate."- [tx opts]- (let [db-version (lowest-dynamic-db-version tx)]+ [tx {:keys [index-metadata] :as opts}]+ (let [db-version (lowest-dynamic-db-version index-metadata tx)](cond(= db-version semantic.db.migration.impl/dynamic-schema-version)(log/info "Dynamic tables migration already performed, skipping.")@@ -107,13 +126,14 @@(defn maybe-migrate!"Execute schema and dynamic schema migrations."- [tx opts]- (ensure-migration-table! tx)+ [tx {:keys [index-metadata] :as opts}]+ (ensure-schema-exists! index-metadata tx)+ (ensure-migration-table! index-metadata tx)(maybe-migrate-schema! tx opts)(maybe-migrate-dynamic-schema! tx opts)nil)(defn drop-migration-table!"Drop migration table."- [connectable]- (jdbc/execute! connectable (sql/format (sql.helpers/drop-table :if-exists :migration))))+ [index-metadata connectable]+ (jdbc/execute! connectable (sql/format (sql.helpers/drop-table :if-exists (migration-table-kw index-metadata)))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex ca535fb0c50..c560223c7ce 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -2,11 +2,13 @@(:require[honey.sql :as sql][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.collections.curation :as collections.curation][metabase.config.core :as config][metabase.util :as u][metabase.util.log :as log][next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs][toucan2.core :as t2]))(def schema-version@@ -14,28 +16,53 @@schema migration will be performed."2)+(def ^:private app-db-sentinel-tables+ "Tables specific to a Metabase application database, chosen to avoid generic names (e.g. Liquibase's+ `databasechangelog`) that a genuinely dedicated pgvector database might also carry.+ Their presence in a to-be-wiped dedicated database means MB_PGVECTOR_DB_URL was pointed at an app db."+ #{"core_user" "metabase_database"})+(defn- drop-all-but-migration-table- [tx]- (let [table-names (map (comp first vals)- (jdbc/execute! tx- (sql/format- {:select [[:tablename :xi]]- :from [:pg_tables]- :where [:and- [:<> :schemaname [:inline "information_schema"]]- [:<> :schemaname [:inline "pg_catalog"]]- [:<> :tablename [:inline "migration"]]]})))]- (doseq [table table-names]+ "Destructive: clears out semantic-search storage ahead of recreating it from scratch.+ When index-metadata carries a `:schema` (shared app-db mode) ONLY tables inside that schema may be+ dropped — the application's tables live in other schemas and must never be touched here.+ Without a `:schema` the database is assumed dedicated to semantic search and its default schema is+ wiped; refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed at+ the application database would otherwise destroy it here, on first init)."+ [index-metadata tx]+ (let [schema (:schema index-metadata)+ tables (jdbc/execute! tx+ (sql/format+ {:select [:schemaname :tablename]+ :from [:pg_tables]+ :where (if schema+ [:and+ [:= :schemaname [:inline schema]]+ [:<> :tablename [:inline "migration"]]]+ ;; dedicated mode only ever creates tables in the default+ ;; schema, so the reset has no business outside it+ [:and+ [:= :schemaname [:raw "current_schema()"]]+ [:<> :tablename [:inline "migration"]]])})+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (when (and (nil? schema)+ (some (comp app-db-sentinel-tables :tablename) tables))+ (throw (ex-info (str "Refusing to reset the semantic search database: it contains Metabase application"+ " tables. Point MB_PGVECTOR_DB_URL at a dedicated pgvector database, or unset it to"+ " share the application database in the isolated semantic_search schema.")+ {:type ::refused-app-db-wipe})))+ (doseq [{:keys [schemaname tablename]} tables](jdbc/execute! tx(sql/format- {:drop-table [[[:raw table]]]})))))+ {:drop-table [[[:raw (str (semantic.util/quote-ident schemaname) "."+ (semantic.util/quote-ident tablename))]]]})))))(defn migrate-schema!"Migrate schema (control, metadata, gate, ...). Migration author is responsible for removing leftovers if necessaryand in general leaving schema in desired state."[tx {:keys [index-metadata] :as _opts}];; ideally index_table indexed are manipulated in dynamic schema part but for now it does not matter- (drop-all-but-migration-table tx)+ (drop-all-but-migration-table index-metadata tx)(semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)(semantic.index-metadata/ensure-control-row-exists! tx index-metadata))@@ -51,18 +78,21 @@in place to be cleaned up or re-created elsewhere."[tx index-metadata target-version alter-fn](let [metadata-table (keyword (:metadata-table-name index-metadata))+ schema (:schema index-metadata)execute! (fn [q] (jdbc/execute! tx (sql/format q)))candidate-names (->> (execute! {:select-distinct [:table_name]:from [metadata-table]:where [[:< :index_version target-version]]})(mapcat vals))- existing-names (when (seq candidate-names)+ ;; stored table_name values are schema-qualified in app-db mode; pg_tables lists bare names+ existing-bare (when (seq candidate-names)(->> (execute! {:select [:tablename]:from [:pg_tables]- :where [:in :tablename (vec candidate-names)]})+ :where (cond-> [:and [:in :tablename (mapv semantic.util/table-name-part candidate-names)]]+ schema (conj [:= :schemaname [:inline schema]]))})(mapcat vals)set))- table-names (filter existing-names candidate-names)]+ table-names (filter (comp existing-bare semantic.util/table-name-part) candidate-names)](when (seq table-names)(doseq [table-name table-names](alter-fn execute! table-name))@@ -122,12 +152,12 @@(fn [execute! table-name](let [kw-tbl (keyword table-name)kw-gate (keyword gate-table)- tbl-model (keyword table-name "model")- tbl-model-id (keyword table-name "model_id")- tbl-root-coll-type (keyword table-name "root_collection_type")- gate-id (keyword gate-table "id")- gate-doc-root [:->> (keyword gate-table "document") [:inline "root_collection_type"]]- composite-gate-id [:|| tbl-model [:inline "_"] tbl-model-id]]+ model-col (semantic.util/column-keyword table-name "model")+ model-id-col (semantic.util/column-keyword table-name "model_id")+ root-coll-type-col (semantic.util/column-keyword table-name "root_collection_type")+ gate-id (semantic.util/column-keyword gate-table "id")+ gate-doc-root [:->> (semantic.util/column-keyword gate-table "document") [:inline "root_collection_type"]]+ composite-gate-id [:|| model-col [:inline "_"] model-id-col]](execute! {:alter-table [kw-tbl] :add-column [[:root_collection_type :text :if-not-exists]]});; Per-row backfill: take whatever the gate document says — authoritative when present.(execute! {:update kw-tbl@@ -135,7 +165,7 @@:set {:root_collection_type gate-doc-root}:where [:and[:= gate-id composite-gate-id]- [:= tbl-root-coll-type nil]+ [:= root-coll-type-col nil][:!= gate-doc-root nil]]});; Forest backfill: one UPDATE per distinct root type, filling rows the gate doc missed.(doseq [[root-type entries] (group-by val root-type-by-coll-id)]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.cljindex 0ec0f9bcdf1..b3e2702cc89 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/dlq.clj@@ -67,8 +67,9 @@sql (sql/format ddl :quoted true)](jdbc/execute-one! pgvector sql));; create attempt_at index- (let [ddl {:create-index [[(keyword (str (name dlq-table) "_attempt_at_idx")) :if-not-exists] [dlq-table :attempt_at]]}- sql (sql/format ddl :quoted true)]+ (let [idx-name (keyword (str (semantic.util/table-name-part (name dlq-table)) "_attempt_at_idx"))+ ddl {:create-index [[idx-name :if-not-exists] [dlq-table :attempt_at]]}+ sql (sql/format ddl :quoted true)](jdbc/execute-one! pgvector sql))nil))@@ -191,7 +192,9 @@[pgvector index-metadata index-id dlq-entries](if (empty? dlq-entries)0- (let [dml {:insert-into (dlq-table-name-kw index-metadata index-id)+ (let [dlq-table (dlq-table-name-kw index-metadata index-id)+ dlq-col (semantic.util/conflict-target-column (name dlq-table) "error_gated_at")+ dml {:insert-into dlq-table:values (sort-by :gate_id dlq-entries):on-conflict [:gate_id]:do-update-set {:fields {:attempt_at :excluded.attempt_at@@ -200,8 +203,7 @@:last_attempted_at :excluded.last_attempted_at};; condition: if exiting dlq entry reflects a more recent gate failure;; prefer the new attempt_at / retry_count from the gate to the prior DLQ entry.- :where [:<= [:. (dlq-table-name-kw index-metadata index-id) :error_gated_at]- :excluded.error_gated_at]}}+ :where [:<= dlq-col :excluded.error_gated_at]}}sql (sql/format dml :quoted true){upsert-count ::jdbc/update-count} (jdbc/execute-one! pgvector sql)](log/debugf "Added/updated %d DLQ entries for index %s" upsert-count index-id)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml b/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.ymlindex e01038b6f3c..a3083464ed7 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/docker-compose.yml@@ -1,3 +1,8 @@+# Dev harness for a DEDICATED pgvector database (MB_PGVECTOR_DB_URL). A dedicated database is not+# required: when MB_PGVECTOR_DB_URL is unset and the application database is Postgres with the pgvector+# extension available, semantic search automatically runs against the app db — its tables live in a+# dedicated `semantic_search` schema and share the app-db connection pool (so heavy reindexing competes+# with regular app-db traffic; use a dedicated database when that matters).version: '3.8'services:diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/env.cljindex ef08ffc6db7..ec73a6e6368 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/env.clj@@ -21,7 +21,10 @@(defn get-index-metadata"Returns this instances index metadata configuration.- Currently, an instance only has one metadata root, so this will always have the same value in production."+ Currently, an instance only has one metadata root, so this will always have the same value in production:+ the schema-qualified configuration when sharing the application database, the bare default otherwise."[];; still useful to indirect this for tests- semantic.index-metadata/default-index-metadata)+ (if (= :app-db (semantic.db.datasource/pgvector-mode))+ semantic.index-metadata/app-db-index-metadata+ semantic.index-metadata/default-index-metadata))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/gate.cljindex e8e4e6ecb5d..434bb682245 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/gate.clj@@ -22,6 +22,7 @@[buddy.core.hash :as buddy-hash][honey.sql :as sql][metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics][metabase.util :as u][metabase.util.json :as json]@@ -122,6 +123,7 @@(peek updates)))seq)](let [{:keys [gate-table-name]} index-metadata+ gate-col (fn [column] (semantic.util/conflict-target-column gate-table-name column))upsert-q {:insert-into (keyword gate-table-name);; sort to ensure locks are acquired predictably:values (sort-by :id gate-document-batch)@@ -134,18 +136,18 @@:gated_at [:clock_timestamp]}:where [:and;; in the case of an updated_at collision, simulate old behaviour, later transactions on the gate win.- [:<= (keyword gate-table-name "updated_at") :excluded.updated_at]+ [:<= (gate-col "updated_at") :excluded.updated_at];; content diff[:case;; now deleted, delete if was not previously[:= nil :excluded.document_hash]- [:!= nil (keyword gate-table-name "document_hash")]+ [:!= nil (gate-col "document_hash")];; was deleted (now has a value) - update- [:= nil (keyword gate-table-name "document_hash")]+ [:= nil (gate-col "document_hash")]true;; update if new value is different:else- [:!= (keyword gate-table-name "document_hash") :excluded.document_hash]]]}}+ [:!= (gate-col "document_hash") :excluded.document_hash]]]}}upsert-sql (sql/format upsert-q :quoted true)](jdbc/execute! tx [(format "SET LOCAL statement_timeout = %d" (.toMillis gate-write-timeout))]) ; note pg cannot accept a parameter here(let [stmt-start (u/start-timer)]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex b51277fface..6f267624373 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -440,9 +440,17 @@(defn- index-name"Returns the name for an index for the given index configuration, column, and index type."[index suffix]- (let [index-name (str (:table-name index) suffix)]+ ;; index names are bare: an index lands in its table's schema and cannot be schema-qualified+ (let [index-name (str (semantic.util/table-name-part (:table-name index)) suffix)](hash-identifier-if-exceeds-pg-limit index-name)))+(defn schema-qualified-index-name+ "The index name qualified with its table's schema (when the table has one), for catalog lookups —+ an index lives in the same schema as its table."+ [index index-name]+ (let [[schema _] (semantic.util/qualified-table-parts (:table-name index))]+ (cond->> index-name schema (str schema "."))))+(defn hnsw-index-name"Returns the name for a HNSW database index for the given semantic search index configuration."[index]@@ -503,7 +511,15 @@(format "whitespace in the table name (%s) is not currently supported" table-name)){:keys [vector-dimensions]} embedding-model](log/info "Creating index table" table-name)- (jdbc/execute! connectable (sql/format (sql.helpers/create-extension :vector :if-not-exists)))+ (try+ (jdbc/execute! connectable (sql/format (sql.helpers/create-extension :vector :if-not-exists)))+ (catch Exception e+ ;; Extension install commonly fails on privileges (may need superuser); give the operator the way out.+ (throw (ex-info (str "Failed to install the pgvector extension. Have a privileged user run"+ " CREATE EXTENSION vector; on this database, or set MB_PGVECTOR_DB_URL to a"+ " database where it is installed.")+ {:type ::extension-install-failed}+ e))))(when force-reset? (drop-index-table! connectable index))(jdbc/execute!connectable@@ -532,7 +548,10 @@[(keyword table-name) :content])sql-format-quoted)))(catch Exception e- (throw (ex-info "Failed to create index table" {} e)))))+ ;; let an already-actionable error (e.g. the pgvector-extension guidance) surface unwrapped+ (if (= ::extension-install-failed (:type (ex-data e)))+ (throw e)+ (throw (ex-info "Failed to create index table" {} e))))))(comment(def embedding-model {:provider "ollama"@@ -1031,13 +1050,17 @@(first decoded)))(defn- find-scan-node- "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`."+ "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`.+ EXPLAIN reports \"Relation Name\" unqualified, so compare on the bare name part."[plan table-name]- (when (map? plan)- (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get plan "Node Type"))- (= table-name (get plan "Relation Name")))- plan- (some #(find-scan-node % table-name) (get plan "Plans")))))+ (let [bare-name (semantic.util/table-name-part table-name)+ search (fn search [node]+ (when (map? node)+ (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get node "Node Type"))+ (= bare-name (get node "Relation Name")))+ node+ (some search (get node "Plans")))))]+ (search plan)))(defn- scan-node-metrics"Pull the index-table scan node out of an EXPLAIN plan tree and summarise it: the plan node chosen, the@@ -1134,7 +1157,7 @@;; path); production traffic leaves it unset and gets the fail-fast.(when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))(not (:vector-search-allow-missing-index? search-context))- (not (semantic.util/index-exists? db (hnsw-index-name index))))+ (not (semantic.util/index-exists? db (schema-qualified-index-name index (hnsw-index-name index)))))(throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. ""Set the semantic-search-vector-strategy setting to an index-backed strategy ""(:hnsw or :hnsw-iterative-*) to build it.")diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.cljindex ad3736e86b6..67aa335c3de 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj@@ -8,6 +8,7 @@[honey.sql :as sql][honey.sql.helpers :as sql.helpers];; TODO: extract schema code to go under db.migration+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.util :as semantic.util][metabase.util :as u]@@ -73,13 +74,29 @@(update :table-name qualify))))(def default-index-metadata- "The default index metadata configuration that will be used for the search engine integration."+ "The index metadata configuration for a dedicated pgvector database (MB_PGVECTOR_DB_URL)."{:version "2":metadata-table-name "index_metadata":control-table-name "index_control":gate-table-name "index_gate":index-table-qualifier "%s"})+(def app-db-index-metadata+ "The index metadata configuration used when semantic search shares the application database.+ Every table lives inside a dedicated schema, so destructive maintenance (see+ [[metabase-enterprise.semantic-search.db.migration.impl]]) is structurally incapable of touching+ application tables.+ Table names are stored and threaded around schema-qualified; derive identifiers and query catalogs via+ [[metabase-enterprise.semantic-search.util/table-name-part]]."+ ;; HoneySQL renders the dotted keywords as schema-qualified identifiers, quoted or not.+ (let [schema semantic.db.datasource/app-db-schema]+ {:version "2"+ :schema schema+ :metadata-table-name (str schema ".index_metadata")+ :control-table-name (str schema ".index_control")+ :gate-table-name (str schema ".index_gate")+ :index-table-qualifier (str schema ".%s")}))+(defn- create-index-metadata-table-if-not-exists-sql [index-metadata](let [{:keys [metadata-table-name]} index-metadataschema (:metadata schema-0)]@@ -151,7 +168,7 @@pgvector(sql/format(sql.helpers/create-index- [(keyword (str (:gate-table-name index-metadata) "_gated_at")) :if-not-exists]+ [(keyword (str (semantic.util/table-name-part (:gate-table-name index-metadata)) "_gated_at")) :if-not-exists][(keyword (:gate-table-name index-metadata)) :gated_at :id]):quoted true))(log/info "Creating gate table tombstone cleanup index if not exists")@@ -160,7 +177,7 @@pgvector(sql/format{:create-index- [[(keyword (str (:gate-table-name index-metadata) "_tombstone_cleanup")) :if-not-exists]+ [[(keyword (str (semantic.util/table-name-part (:gate-table-name index-metadata)) "_tombstone_cleanup")) :if-not-exists][(keyword (:gate-table-name index-metadata)) :gated_at]]:where [:and [:= :document nil] [:= :document_hash nil]]}:quoted true))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.cljindex eec0e025430..da233f0d664 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj@@ -11,6 +11,7 @@[honey.sql.helpers :as sql.helpers][java-time.api :as t][medley.core :as m]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.util :as u][metabase.util.log :as log][metabase.util.string :as u.str]@@ -40,20 +41,24 @@but are not in the repair table. These represent lost deletes."[pgvector gate-table-name repair-table-name](try- (let [anti-join-sql (-> (sql.helpers/select :model :model_id)+ (let [column semantic.util/column-keyword+ anti-join-sql (-> (sql.helpers/select :model :model_id)(sql.helpers/from (keyword gate-table-name))(sql.helpers/where [:not [:exists(-> (sql.helpers/select 1)(sql.helpers/from (keyword repair-table-name))(sql.helpers/where [:and- [:= (keyword repair-table-name "model")- (keyword gate-table-name "model")]- [:= (keyword repair-table-name "model_id")- (keyword gate-table-name "model_id")]]))]])+ [:= (column repair-table-name "model")+ (column gate-table-name "model")]+ [:= (column repair-table-name "model_id")+ (column gate-table-name "model_id")]]))]])(sql/format :quoted true))results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})](log/infof "Found %d documents in gate table that should be deleted" (count results))results)+ ;; TODO (Chris 2026-07-13) -- swallowing the error reads as "no lost deletes", so a transient anti-join+ ;; failure silently skips tombstoning that run. Tolerated because the next repair pass retries; revisit+ ;; if lost deletes start slipping through.(catch Exception e(log/errorf e "Error finding lost deletes between gate table %s and repair table %s"gate-table-name repair-table-name))))@@ -89,18 +94,20 @@(log/warnf e "Failed to drop repair table: %s" repair-table-name))))(defn- repair-table-name- "Generates a unique name for a repair table with timestamp for cleanup.+ "Generates a unique name for a repair table with timestamp for cleanup, qualified like the index tables+ so it lands in the module's schema when sharing the app db.Format: repair_<millis-since-epoch>_<short-id>"- []+ [index-metadata](let [millis-since-epoch (t/to-millis-from-epoch (t/instant))short-id (u/lower-case-en (u.str/random-string 6))]- (format "repair_%d_%s" millis-since-epoch short-id)))+ (format (:index-table-qualifier index-metadata "%s")+ (format "repair_%d_%s" millis-since-epoch short-id))))(defn with-repair-table!"Creates a repair table, executes a function with the table name, and ensures cleanup.Returns the result of calling f with the repair table name."- [pgvector f]- (let [repair-table (repair-table-name)]+ [pgvector index-metadata f]+ (let [repair-table (repair-table-name index-metadata)](try(create-repair-table! pgvector repair-table)(f repair-table)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.cljindex f382d11680b..d684057d4b6 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj@@ -21,21 +21,48 @@(set! *warn-on-reflection* true)+;; Both orphan sweeps feed table names to DROP, so in shared app-db mode the schema scoping keeps a LIKE+;; pattern from matching and dropping application tables.++(defn- scope-where-to-schema+ "Restrict an `information_schema.tables` WHERE (an aliased-`:t` `[:and ...]` vector) to the module schema.+ A no-op without a schema (dedicated mode)."+ [where schema]+ (cond-> where schema (conj [:= :t.table_schema [:inline schema]])))++(defn- requalify-table-names+ "Prefix bare `information_schema` table names with the module schema so they match how the metadata table+ stores them. A no-op in dedicated mode."+ [schema table-names]+ (map #(cond->> % schema (str schema ".")) table-names))+(defn- orphan-index-tables"Returns a list of semantic search index tables which are not referenced in the metadata table.- Not expected to occur, but if it does, these tables can be dropped."- [pgvector {:keys [metadata-table-name]}]- (let [orphaned-tables-sql+ Not expected to occur, but if it does, these tables can be dropped.+ With a `:schema` (shared app-db mode) only tables inside the module's schema are considered, and the+ returned names are schema-qualified to match how the metadata table stores them."+ ;; TODO (Chris 2026-07-09) -- BOT-1833: the LIKE 'index_table_%' pattern predates the+ ;; index_<provider>_<model>_<dims> naming, so it matches nothing and orphan detection is a no-op. A fix+ ;; needs shape-aware matching that excludes index_metadata/index_control/index_gate (not registered in+ ;; metadata.table_name — a naive 'index\_%' pattern would drop them as orphans).+ [pgvector {:keys [metadata-table-name schema]}]+ (let [;; information_schema reports bare names; stored metadata names are qualified when we have a schema+ stored-name (if schema+ [:|| [:inline (str schema ".")] :t.table_name]+ :t.table_name)+ orphaned-tables-sql(-> {:select [:t.table_name]:from [[:information_schema.tables :t]]:left-join [[(keyword metadata-table-name) :meta]- [:= :meta.table_name :t.table_name]]- :where [:and- [:like :t.table_name [:inline "index_table_%"]]- [:= :meta.table_name nil]]}+ [:= :meta.table_name stored-name]]+ :where (scope-where-to-schema [:and+ [:like :t.table_name [:inline "index_table_%"]]+ [:= :meta.table_name nil]]+ schema)}(sql/format :quoted true))](->> (jdbc/execute! pgvector orphaned-tables-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (map :table_name))))+ (map :table_name)+ (requalify-table-names schema))))(defn- parse-repair-table-timestamp"Extracts timestamp from repair table name. Returns nil if parsing fails."@@ -51,19 +78,22 @@Repair tables are named: repair_<millis-since-epoch>_<short-id>Repair tables should not become orphaned under normal operation, since they should be deleted immeditaely after use.- However, in case of a crash or failure, they may be left behind, so we clean them up after a retention period."- [pgvector]+ However, in case of a crash or failure, they may be left behind, so we clean them up after a retention period.+ With a `:schema` (shared app-db mode) only tables inside the module's schema are considered — a `repair_%`+ pattern must never match application tables — and the returned names are schema-qualified."+ [pgvector {:keys [schema]}](let [retention-cutoff (t/minus (t/instant) (t/hours (semantic.settings/repair-table-retention-hours)))repair-tables-sql (-> {:select [:t.table_name]:from [[:information_schema.tables :t]]- :where [:like :t.table_name [:inline "repair_%"]]}+ :where (scope-where-to-schema [:and [:like :t.table_name [:inline "repair_%"]]] schema)}(sql/format :quoted true))all-repair-tables (->> (jdbc/execute! pgvector repair-tables-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})(map :table_name))- old-tables (filter (fn [table-name]- (when-let [table-timestamp (parse-repair-table-timestamp table-name)]- (t/before? table-timestamp retention-cutoff)))- all-repair-tables)]+ old-tables (->> all-repair-tables+ (filter (fn [table-name]+ (when-let [table-timestamp (parse-repair-table-timestamp table-name)]+ (t/before? table-timestamp retention-cutoff))))+ (requalify-table-names schema))](when (seq old-tables)(log/infof "Found %d orphaned repair tables older than %d hours"(count old-tables)@@ -72,8 +102,8 @@(defn- cleanup-orphan-repair-tables!"Cleans up repair tables that are older than the retention period."- [pgvector]- (let [orphan-tables (orphan-repair-tables pgvector)]+ [pgvector index-metadata]+ (let [orphan-tables (orphan-repair-tables pgvector index-metadata)](when (seq orphan-tables)(let [tables-to-drop (map keyword orphan-tables)drop-table-sql (sql/format@@ -168,13 +198,17 @@(when (semantic.u/semantic-search-available?)(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.env/get-index-metadata)]- ;; Available but never initialized is a steady state (semantic neither default nor additional);- ;; the bookkeeping tables only exist after init, and the cleanup queries would throw without them.+ ;; Available but never initialized is a steady state (semantic neither default nor additional,+ ;; or app-db mode with the engine disabled); the bookkeeping tables only exist after init, and+ ;; the cleanup queries would throw without them.;; Still runs while merely available, so an opted-out instance's leftover indexes get cleaned.(when (semantic.index-metadata/control-and-metadata-tables-exist? pgvector index-metadata)(cleanup-stale-indexes! pgvector index-metadata)- (cleanup-old-gate-tombstones! pgvector index-metadata)- (cleanup-orphan-repair-tables! pgvector)))))+ (cleanup-old-gate-tombstones! pgvector index-metadata))+ ;; Repair-table cleanup is information_schema-driven, so it needs no bookkeeping tables and runs+ ;; outside the guard above. Shared app-db mode scopes the sweep to the module schema; dedicated mode+ ;; owns the whole DB, so its `repair_%` sweep is DB-wide by design (a mispointed URL is the caller's).+ (cleanup-orphan-repair-tables! pgvector index-metadata))))(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex a0c9c733650..09452dd8efe 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -51,9 +51,11 @@;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY;; registers the index in pg_indexes as soon as the build starts, so this fires only once.- (when (and (hnsw-strategy?)- (not (semantic.u/index-exists? pgvector (semantic.index/hnsw-index-name index))))- (semantic.core/build-hnsw-index-async!))+ (let [hnsw-index (semantic.index/schema-qualified-index-name+ index (semantic.index/hnsw-index-name index))]+ (when (and (hnsw-strategy?)+ (not (semantic.u/index-exists? pgvector hnsw-index)))+ (semantic.core/build-hnsw-index-async!)))(semantic-search.indexer/quartz-job-run! pgvector index-metadata));; Engines can activate at runtime (license applied, kill switch re-enabled,;; additional-search-engines set on another node); initializing from the next tick healsdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 69fe3590a3b..055c32d6a66 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -1,46 +1,97 @@(ns metabase-enterprise.semantic-search.util(:require+ [clojure.string :as str][metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase.app-db.core :as mdb][metabase.premium-features.core :as premium-features][metabase.search.engine :as search.engine][next.jdbc :as jdbc]+ [next.jdbc.quoted :as quoted][next.jdbc.result-set :as jdbc.rs]))+(def quote-ident+ "Quote a Postgres identifier for interpolation into raw SQL, doubling any embedded double quote."+ quoted/postgres)++(defn qualified-table-parts+ "Split a possibly schema-qualified table name into `[schema table]`; `schema` is nil when unqualified."+ [table-name]+ (let [[a b] (str/split table-name #"\." 2)]+ (if b [a b] [nil a])))++(defn table-name-part+ "The bare table name of a possibly schema-qualified table name.+ Derived identifiers (index names, catalog lookups by `tablename`) must use this, never the full name."+ [table-name]+ (second (qualified-table-parts table-name)))++(defn column-keyword+ "A `table.column` reference as a dotted-name keyword, not a namespaced one.+ HoneySQL quotes a keyword's namespace as one identifier, so a schema-qualified table there renders as a+ single broken identifier. The dotted name renders as separate quoted parts.+ Pass the table name as it should render: qualified for an ordinary FROM/JOIN reference."+ [table-name column]+ (keyword (str table-name "." (name column))))++(defn conflict-target-column+ "A `column-keyword` for an ON CONFLICT clause on a possibly schema-qualified `table-name`.+ Postgres names the conflict target by its bare relation even when the insert target is schema-qualified,+ so the reference drops the schema."+ [table-name column]+ (column-keyword (table-name-part table-name) column))+(defn table-exists?- "Does the table-name exist in pgvector DB's information_schema.tables?"+ "Does the table-name exist in pgvector DB's information_schema.tables?+ A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."[pgvector table-name]- (-> (jdbc/execute-one! pgvector- ["SELECT exists (select 1 FROM information_schema.tables WHERE table_name = ?) table_exists"- table-name]- {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (:table_exists false)))+ (let [[schema table] (qualified-table-parts table-name)]+ (-> (jdbc/execute-one! pgvector+ (if schema+ [(str "SELECT exists (select 1 FROM information_schema.tables"+ " WHERE table_schema = ? AND table_name = ?) table_exists")+ schema table]+ ["SELECT exists (select 1 FROM information_schema.tables WHERE table_name = ?) table_exists"+ table])+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (:table_exists false))))(defn index-exists?- "Does an index named `index-name` exist in the pgvector DB's pg_indexes?"+ "Does an index named `index-name` exist in the pgvector DB's pg_indexes?+ A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."[pgvector index-name]- (-> (jdbc/execute-one! pgvector- ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"- index-name]- {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (:index_exists false)))+ (let [[schema index] (qualified-table-parts index-name)]+ (-> (jdbc/execute-one! pgvector+ (if schema+ ["SELECT exists (select 1 FROM pg_indexes WHERE schemaname = ? AND indexname = ?) index_exists"+ schema index]+ ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"+ index])+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (:index_exists false))))(defn semantic-search-configured?- "Is a pgvector DB configured for this instance?- The boot-static input: gates Quartz job scheduling, which happens once at startup.- The runtime gates (license, kill switch, engine activity) are checked per job execution instead, so- flipping them never requires a restart.- This check becomes a DB probe under BOT-1796."+ "Whether to schedule the semantic-search Quartz jobs at startup.+ True when the `:semantic-search` feature is present and a pgvector store might exist: a dedicated+ MB_PGVECTOR_DB_URL, or a Postgres app DB that [[semantic-search-capable?]] can probe to answer for sure.+ Cheap and infallible by contract -- it runs at boot and never queries the DB."[]- (string? (not-empty semantic.db.datasource/db-url)))+ ;; The license is in this boot gate, not only the per-execution gates, so an unlicensed instance's+ ;; scheduler stays free of no-op jobs. The asymmetry is deliberate: removing the feature at runtime lets+ ;; the scheduled jobs no-op via semantic-search-active?, but adding it needs a restart before they+ ;; schedule. The kill switch and engine activity stay per-execution so they never need one.+ (and (premium-features/has-feature? :semantic-search)+ (or (semantic.db.datasource/dedicated-url-configured?)+ (= :postgres (mdb/db-type)))))(defn semantic-search-capable?"Does this instance have the infrastructure for semantic search: the premium feature and a pgvector DB.Deliberately excludes the kill switch, which is checked per execution so it works at runtime."[]- ;; Cheap gate first: semantic-search-configured? becomes a DB probe under BOT-1796.+ ;; Feature first: the pgvector check may probe the app DB, and instances that can't use the answer+ ;; must never probe.(and (premium-features/has-feature? :semantic-search)- (semantic-search-configured?)))+ (semantic.db.datasource/pgvector-configured?)))(defn semantic-search-available?"Whether semantic search can run on this instance: capable and not disabled by the kill switch.diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.cljindex 7f7ea85b3ff..472d0f20064 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj@@ -10,10 +10,15 @@[metabase.entity-retrieval.mirror :as mirror][metabase.metabot.tools.entity-retrieval :as tools.entity-retrieval][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][next.jdbc :as jdbc]))(set! *warn-on-reflection* true)+;; the dedicated-harness tests below hit the app db before any auto-initializing mt helper — on the+;; appdb-mode CI job this namespace can be an early db touch in a fresh JVM+(use-fixtures :once (fixtures/initialize :db :test-users))+(defn- approx [target] #(< (abs (- (double %) (double target))) 1e-9))(deftest score-shape-matches-regular-search-test@@ -34,7 +39,8 @@(deftest dispatch-without-pgvector-test(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully";; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:- ;; available? is false, so search returns [] and the write-path nudge no-ops rather than throwing.+ ;; entity retrieval is dedicated-only, so no URL means available? is false, search returns [] and the+ ;; write-path nudge no-ops rather than throwing.(mt/with-premium-features #{:library :library-retrieval}(with-redefs [semantic.db.datasource/db-url nil](is (= [] (mirror/search "anything" 10)))@@ -134,9 +140,11 @@":library without :library-retrieval does not entitle the tool"))(mt/with-premium-features #{:library :library-retrieval}(is (true? (entity-retrieval.core/available?)))))- ;; pgvector unconfigured -> unavailable regardless of license+ ;; no URL -> unconfigured and unavailable regardless of license, even on a pgvector-capable+ ;; Postgres app db: entity retrieval's tables aren't schema-isolated yet, so it stays dedicated-only(with-redefs [semantic.db.datasource/db-url nil](mt/with-premium-features #{:library :library-retrieval}+ (is (false? (entity-retrieval.core/pgvector-configured?)))(is (false? (entity-retrieval.core/available?))))))(testing "fully licensed + pgvector, but no way to compute embeddings -> unavailable"(with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.cljindex 4f94d1300c9..fbc949501b0 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj@@ -10,12 +10,17 @@[metabase.measures.test-util :as measures.tu][metabase.metabot.tools.search :as tools.search][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][next.jdbc :as jdbc][next.jdbc.result-set :as jdbc.rs][toucan2.core :as t2]))(set! *warn-on-reflection* true)+;; raw t2/collections.tu access below runs before any auto-initializing mt helper, so the app db must be+;; set up explicitly — on the appdb-mode CI job this namespace can be the first db touch in the JVM+(use-fixtures :once (fixtures/initialize :db :test-users))+;; OsiAiContext write hooks fire request-entity-sync!, which would spawn a background targeted reconcile;; using the *real* configured embedding model and race these tests — they drive reconcile! / reconcile-entity!;; with the mock model directly. Suppress the nudge here; the hook -> drain path is covered in core-test.diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.cljindex 812f29e6ea0..cc4dd965337 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj@@ -3,7 +3,6 @@(:require[clojure.test :refer :all][metabase-enterprise.semantic-search.test-util :as semantic.tu]- [metabase.api.common :as api][metabase.metabot.tools.search :as search][metabase.permissions.core :as perms][metabase.search.core :as search-core]@@ -27,82 +26,101 @@`(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))(deftest search-test+ ;; Fully mocked: each sub-test redefs search-core/search, so no engine or index runs. active-engines is+ ;; pinned to include semantic so search/search takes the two-engine fusion branch (term queries via the+ ;; fallback engine, semantic queries via semantic), regardless of whether this instance has pgvector.(mt/with-additional-premium-features #{:content-verification}- (with-semantic-search-if-available!- (mt/with-test-user :rasta- (let [order-table {:id 1- :model "table"- :table_name "orders"- :name "Orders"- :description "Order table"- :database_id 42- :table_schema "public"}- dashboard {:id 2- :model "dashboard"- :name "Sales Dashboard"- :description "Dashboard for sales"- :verified true}]- (with-redefs [perms/impersonated-user? (fn [] false)- perms/sandboxed-user? (fn [] false)- api/*current-user-id* 1]- (testing "search returns postprocessed results for term queries"- (with-redefs [search-core/search (fn [_] {:data [order-table]})]- (let [args {:term-queries ["orders"]- :entity-types ["table"]}- results (search/search args)- expected [(#'search/postprocess-search-result order-table)]]- (is (= expected results)))))- (testing "search returns postprocessed results for semantic queries"- (with-redefs [search-core/search (fn [_] {:data [dashboard]})]- (let [args {:semantic-queries ["sales metrics"]- :entity-types ["dashboard"]}- results (search/search args)- expected [(#'search/postprocess-search-result dashboard)]]- (is (= expected results)))))- (testing "search combines term and semantic queries using RRF"- (with-redefs [search-core/search (fn [context]- (if (= (:search-string context) "orders")- {:data [order-table]}- {:data [dashboard]}))]- (let [args {:term-queries ["orders"]- :semantic-queries ["sales"]- :entity-types ["table" "dashboard"]}- results (search/search args)]- ;; Should return both results combined via RRF- (is (= 2 (count results)))- (is (some #(= (:id %) 1) results))- (is (some #(= (:id %) 2) results)))))- (testing "search applies RRF to overlapping results"- (with-redefs [search-core/search (fn [_]- {:data [order-table dashboard]})]- (let [args {:term-queries ["orders" "sales"]- :entity-types ["table" "dashboard"]}- results (search/search args)]- ;; Both queries return same results, RRF should boost them- (is (= 2 (count results)))- (is (some #(= (:id %) 1) results))- (is (some #(= (:id %) 2) results)))))- (testing "search handles empty results"- (with-redefs [search-core/search (fn [_] {:data []})]- (let [args {:term-queries ["nonexistent"]- :entity-types ["table"]}- results (search/search args)]- (is (empty? results)))))- (testing "search with metabot verified-or-curated content flag"- (let [metabot {:entity_id "test-bot"- :use_verified_content true}]- (with-redefs [t2/select-one (fn [model & _]- (is (= :model/Metabot model) "Should query for Metabot model")- metabot)- search-core/search (fn [context]- ;; use_verified_content now drives the curated filter, not :verified- (is (true? (:curated? context)))- {:data [dashboard]})]- (let [results (search/search {:term-queries ["test"]- :metabot-id "test-bot"- :entity-types ["dashboard"]})]- (is (= 1 (count results)))- (is (= 2 (:id (first results))))))))))))))+ (mt/with-test-user :rasta+ (let [order-table {:id 1+ :model "table"+ :table_name "orders"+ :name "Orders"+ :description "Order table"+ :database_id 42+ :table_schema "public"}+ dashboard {:id 2+ :model "dashboard"+ :name "Sales Dashboard"+ :description "Dashboard for sales"+ :verified true}]+ (with-redefs [perms/impersonated-user? (fn [] false)+ perms/sandboxed-user? (fn [] false)+ search.engine/active-engines (constantly [:search.engine/semantic :search.engine/appdb])]+ (testing "search returns postprocessed results for term queries"+ (with-redefs [search-core/search (fn [_] {:data [order-table]})]+ (let [args {:term-queries ["orders"]+ :entity-types ["table"]}+ results (search/search args)+ expected [(#'search/postprocess-search-result order-table)]]+ (is (= expected results)))))+ (testing "search returns postprocessed results for semantic queries"+ (with-redefs [search-core/search (fn [_] {:data [dashboard]})]+ (let [args {:semantic-queries ["sales metrics"]+ :entity-types ["dashboard"]}+ results (search/search args)+ expected [(#'search/postprocess-search-result dashboard)]]+ (is (= expected results)))))+ (testing "search combines term and semantic queries using RRF"+ (with-redefs [search-core/search (fn [context]+ (if (= (:search-string context) "orders")+ {:data [order-table]}+ {:data [dashboard]}))]+ (let [args {:term-queries ["orders"]+ :semantic-queries ["sales"]+ :entity-types ["table" "dashboard"]}+ results (search/search args)]+ ;; Should return both results combined via RRF+ (is (= 2 (count results)))+ (is (some #(= (:id %) 1) results))+ (is (some #(= (:id %) 2) results)))))+ (testing "with no semantic engine active, semantic queries fold into the default engine"+ (with-redefs [search.engine/active-engines (constantly [:search.engine/appdb])+ ;; Pass terms through uncombined: Postgres appdb would OR them into one string,+ ;; which breaks the per-query mock keying below. Keeps this deterministic on any app-db.+ search.engine/disjunction (fn [_ terms] terms)+ search-core/search (fn [context]+ (if (= (:search-string context) "orders")+ {:data [order-table]}+ {:data [dashboard]}))]+ (let [args {:term-queries ["orders"]+ :semantic-queries ["sales"]+ :entity-types ["table" "dashboard"]}+ results (search/search args)]+ ;; The semantic query is not dropped — it is searched via the default engine and fused in.+ (is (= 2 (count results)))+ (is (some #(= (:id %) 1) results))+ (is (some #(= (:id %) 2) results)))))+ (testing "search applies RRF to overlapping results"+ (with-redefs [search-core/search (fn [_]+ {:data [order-table dashboard]})]+ (let [args {:term-queries ["orders" "sales"]+ :entity-types ["table" "dashboard"]}+ results (search/search args)]+ ;; Both queries return same results, RRF should boost them+ (is (= 2 (count results)))+ (is (some #(= (:id %) 1) results))+ (is (some #(= (:id %) 2) results)))))+ (testing "search handles empty results"+ (with-redefs [search-core/search (fn [_] {:data []})]+ (let [args {:term-queries ["nonexistent"]+ :entity-types ["table"]}+ results (search/search args)]+ (is (empty? results)))))+ (testing "search with metabot verified-or-curated content flag"+ (let [metabot {:entity_id "test-bot"+ :use_verified_content true}]+ (with-redefs [t2/select-one (fn [model & _]+ (is (= :model/Metabot model) "Should query for Metabot model")+ metabot)+ search-core/search (fn [context]+ ;; use_verified_content now drives the curated filter, not :verified+ (is (true? (:curated? context)))+ {:data [dashboard]})]+ (let [results (search/search {:term-queries ["test"]+ :metabot-id "test-bot"+ :entity-types ["dashboard"]})]+ (is (= 1 (count results)))+ (is (= 2 (:id (first results)))))))))))));; Mock embeddings: similar vectors for semantic synonym pairs, orthogonal vectors for unrelated terms.(def ^:private test-mock-embeddingsdiff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.cljnew file mode 100644index 00000000000..9a81ec992f1--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj@@ -0,0 +1,135 @@+(ns metabase-enterprise.semantic-search.appdb-pgvector-mode-test+ "End-to-end round trip for pgvector-on-the-app-db mode: no MB_PGVECTOR_DB_URL, semantic search runs+ against the application database inside the semantic_search schema, sharing the app-db pool.++ Opt-in and self-gated: the destructive round trip (installs the vector extension, drops the+ semantic_search schema on cleanup) only runs with MB_APPDB_PGVECTOR_MODE_TEST=true — set by the+ appdb-mode CI job, whose pgvector/pgvector app-db image it then REQUIRES (skips fail loudly there).+ Deliberately does NOT use test-util's once-fixture — that fixture gates on the dedicated-harness+ MB_PGVECTOR_DB_URL, which is exactly what this test runs without."+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing use-fixtures]]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.app-db.core :as mdb]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]))++(set! *warn-on-reflection* true)++;; the mode probe deliberately answers only after the app db is set up (mdb/db-is-set-up?), and this+;; namespace can be the first thing a fresh test JVM runs+(use-fixtures :once (fixtures/initialize :db :test-users))++(deftest ^:parallel get-index-metadata-mode-test+ (testing "get-index-metadata returns the schema-qualified config exactly in app-db mode"+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/pgvector-mode (constantly :app-db)]+ (is (= semantic.index-metadata/app-db-index-metadata (semantic.env/get-index-metadata))))+ (doseq [mode [:dedicated :unavailable]]+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/pgvector-mode (constantly mode)]+ (is (= semantic.index-metadata/default-index-metadata (semantic.env/get-index-metadata)))))))++(defn- appdb-can-host-pgvector?+ "Real-environment gate for the round trip: a Postgres app db where the vector extension is installed or+ installable. Read-only — activation (init-semantic-search!) installs the extension and creates the+ schema."+ []+ (and (= :postgres (mdb/db-type))+ (try+ (semantic.db.datasource/check-app-db-pgvector-support)+ (catch Exception _ false))))++(defn- tables-in-schema+ [connectable schema]+ (->> (jdbc/execute! connectable+ ["SELECT tablename FROM pg_tables WHERE schemaname = ?" schema]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (map :tablename)+ set))++(defn- schema-exists?+ [connectable schema]+ (boolean (seq (jdbc/execute! connectable+ ["SELECT 1 FROM information_schema.schemata WHERE schema_name = ?" schema]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))++(defn- opted-in?+ "True when MB_APPDB_PGVECTOR_MODE_TEST=true: the appdb-mode CI job (or a developer deliberately opting+ in). The round trip installs the vector extension and drops the semantic_search schema on cleanup, so it+ never runs implicitly against whatever Postgres app db happens to be around."+ []+ (Boolean/parseBoolean (System/getenv "MB_APPDB_PGVECTOR_MODE_TEST")))++(deftest ^:synchronized appdb-pgvector-round-trip-test+ (cond+ (not (opted-in?))+ (testing "destructive round trip requires the MB_APPDB_PGVECTOR_MODE_TEST opt-in — skipping"+ (is true))++ (not (appdb-can-host-pgvector?))+ (testing "opted in, but the app db can't host pgvector — the CI service must be misconfigured"+ (is false "the appdb-mode job expects a pgvector-capable Postgres app db"))++ ;; the app db already carries a semantic_search schema (a real app-db-mode deployment, or even an empty+ ;; schema) — this test's cleanup DROP SCHEMA CASCADE would clobber it, so refuse rather than destroy it+ (schema-exists? (mdb/data-source) "semantic_search")+ (testing "opted in, but a semantic_search schema already exists — refusing to clobber it"+ (is false "the appdb-mode job should start from a fresh app db with no semantic_search schema"))++ :else+ (mt/with-premium-features #{:semantic-search}+ ;; the pair mirrors mock-embeddings' doc/query geometry: nearly identical vectors, so the query+ ;; lands well inside the distance threshold for its document and nothing else+ (semantic.tu/with-mock-embeddings {"Dog Training Guide" [0.12 -0.34 0.56 -0.78]+ "puppy" [0.13 -0.33 0.57 -0.77]}+ (with-redefs [semantic.db.datasource/db-url nil+ semantic.db.datasource/data-source (atom nil)+ ;; a cooldown left open by another test would make pgvector-mode read :unavailable here+ semantic.db.datasource/app-db-pgvector-support (atom nil)+ semantic.db.datasource/probe-cooldown-timer (atom nil)+ semantic.db.datasource/logged-pgvector-absent? (atom false)+ semantic.embedding/get-configured-model (fn [] semantic.tu/mock-embedding-model)]+ (let [app-db (mdb/data-source)]+ (try+ (testing "activation is automatic: Postgres app db + extension, no URL"+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))+ (is (semantic.db.datasource/pgvector-configured?)))+ (testing "the module uses the shared app-db pool and the schema-qualified config"+ (is (identical? app-db (semantic.env/get-pgvector-datasource!)))+ (is (nil? @semantic.db.datasource/data-source)+ "the shared pool must never land in the module's own pool atom")+ (is (= "semantic_search" (:schema (semantic.env/get-index-metadata)))))+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)+ public-before (tables-in-schema app-db "public")]+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata+ (semantic.env/get-configured-embedding-model))+ (semantic.pgvector-api/index-documents! pgvector index-metadata (semantic.tu/mock-documents))+ (testing "every module table lives inside the semantic_search schema"+ (let [tables (tables-in-schema app-db "semantic_search")]+ (is (contains? tables "migration"))+ (is (contains? tables "index_metadata"))+ (is (contains? tables "index_control"))+ (is (contains? tables "index_gate"))+ (is (contains? tables "index_mock_model_4"))+ (is (some #(str/starts-with? % "dlq_") tables))))+ (testing "search round-trips through the app-db pool"+ ;; results are reconstructed from legacy_input, so the mock card comes back as model+id+ (is (= [{:model "card" :id 123}]+ (->> (mt/with-test-user :crowberto+ (semantic.pgvector-api/query pgvector index-metadata+ {:search-string "puppy" :archived? false}))+ :results+ (filter (comp #{"card"} :model))+ (mapv #(select-keys % [:model :id]))))))+ (testing "the application schema is untouched"+ (is (= public-before (tables-in-schema app-db "public")))))+ (finally+ (jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"])))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.cljindex 534e04e5e58..f6c64514d7a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj@@ -2,9 +2,11 @@(:require[clojure.test :refer :all][metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics][metabase.app-db.core :as mdb][metabase.search.appdb.core :as appdb]@@ -16,6 +18,18 @@(use-fixtures :once #'semantic.tu/once-fixture)+(deftest supported?-requires-an-embedder-test+ (testing "a pgvector store alone is not enough to select the engine — auto-activation without a+ configured embedding backend would loop every index and query embed into failure"+ (mt/with-premium-features #{:semantic-search}+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-available? (constantly true)]+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly {:provider "no-such-provider"})]+ (is (false? (semantic.core/supported?))))+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly semantic.tu/mock-embedding-model)]+ (is (true? (semantic.core/supported?))))))))+(deftest fallback-engine-available-with-semantic-test(mt/with-premium-features #{:semantic-search}(mt/with-temporary-setting-values [search.settings/search-engine "semantic"]@@ -40,7 +54,10 @@reset-called-atoms! #(do (reset! semantic-called? false)(reset! appdb-called? false)(reset! legacy-called? false))]- (mt/with-dynamic-fn-redefs [semantic.pgvector-api/query+ (mt/with-dynamic-fn-redefs [;; supported? requires a configured embedder; pin the mock so semantic is selectable+ semantic.embedding/get-configured-model+ (constantly semantic.tu/mock-embedding-model)+ semantic.pgvector-api/query(fn [& _](reset! semantic-called? true){:results [{:id 1 :name "semantic-result" :model "card" :collection_id 1 :score 0}]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.cljnew file mode 100644index 00000000000..ef0b572bd0c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj@@ -0,0 +1,200 @@+(ns metabase-enterprise.semantic-search.db.datasource-mode-test+ "Unit tests (redefs only, no databases) for pgvector mode selection: dedicated URL vs. shared app-db vs.+ unavailable. The end-to-end app-db mode round-trip lives in+ metabase-enterprise.semantic-search.appdb-pgvector-mode-test."+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.app-db.core :as mdb]+ [metabase.test :as mt]+ [next.jdbc :as jdbc])+ (:import+ (clojure.lang ExceptionInfo)))++(set! *warn-on-reflection* true)++(defmacro ^:private with-support-cache+ "Run body with all three pieces of app-db probe state rebound to fresh atoms: the support cache (holding+ `init`), the cooldown timer, and the log-once hint latch."+ [init & body]+ `(with-redefs [semantic.db.datasource/app-db-pgvector-support (atom ~init)+ semantic.db.datasource/probe-cooldown-timer (atom nil)+ semantic.db.datasource/logged-pgvector-absent? (atom false)]+ ~@body))++(deftest dedicated-mode-wins-test+ (testing "MB_PGVECTOR_DB_URL always wins, even when the app db could also support pgvector"+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://localhost:5432/pgvector"+ mdb/db-type (constantly :postgres)]+ (with-support-cache true+ (is (= :dedicated (semantic.db.datasource/pgvector-mode)))+ (is (semantic.db.datasource/pgvector-configured?))))))++(deftest whitespace-url-counts-as-unset-test+ (testing "a whitespace-only MB_PGVECTOR_DB_URL is unset for every predicate — no silent app-db fallback+ while a task gate elsewhere believes a dedicated store is configured"+ (with-redefs [semantic.db.datasource/db-url " "+ mdb/db-type (constantly :h2)]+ (with-support-cache nil+ (is (false? (semantic.db.datasource/dedicated-url-configured?)))+ (is (false? (semantic.util/semantic-search-configured?)))+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))))))++(deftest non-postgres-app-db-test+ (testing "no URL + non-Postgres app db → :unavailable without ever probing the app db"+ (doseq [db-type [:h2 :mysql]]+ (with-redefs [semantic.db.datasource/db-url nil+ mdb/db-type (constantly db-type)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not probe a non-Postgres app db")))]+ (with-support-cache nil+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))+ (is (not (semantic.db.datasource/pgvector-configured?)))+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))))))++(deftest support-requires-provisionable-store-test+ (testing "supported only when the vector extension and the semantic_search schema exist or can be created"+ (letfn [(check [] (semantic.db.datasource/check-app-db-pgvector-support))+ (catalog [m] (fn [& _] m))]+ (with-redefs [mdb/data-source (constantly ::app-pool)]+ (testing "extension neither installed nor available → unsupported, no provisioning probe"+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available false :schema-exists false})+ semantic.db.datasource/app-db-can-provision-pgvector?+ (fn [& _] (throw (AssertionError. "must not probe when the extension is unavailable")))]+ (is (false? (check)))))+ (testing "extension installed and schema present → supported without a DDL probe"+ (with-redefs [jdbc/execute-one! (catalog {:installed true :available true :schema-exists true})+ semantic.db.datasource/app-db-can-provision-pgvector?+ (fn [& _] (throw (AssertionError. "must not probe when already fully provisioned")))]+ (is (true? (check)))))+ (testing "installed but schema missing → probe schema creation only, not the extension"+ (with-redefs [jdbc/execute-one! (catalog {:installed true :available true :schema-exists false})+ semantic.db.datasource/app-db-can-provision-pgvector?+ (fn [_ create-extension? create-schema?]+ (is (= [false true] [create-extension? create-schema?]))+ true)]+ (is (true? (check)))))+ (testing "available but not installed → probe both extension and schema creation"+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists false})+ semantic.db.datasource/app-db-can-provision-pgvector?+ (fn [_ create-extension? create-schema?]+ (is (= [true true] [create-extension? create-schema?]))+ true)]+ (is (true? (check)))))+ (testing "available but not installed, schema pre-created → probe the extension only"+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists true})+ semantic.db.datasource/app-db-can-provision-pgvector?+ (fn [_ create-extension? create-schema?]+ (is (= [true false] [create-extension? create-schema?]))+ true)]+ (is (true? (check)))))+ (testing "a privilege gap while provisioning → unsupported"+ (with-redefs [jdbc/execute-one! (catalog {:installed false :available true :schema-exists false})+ semantic.db.datasource/app-db-can-provision-pgvector? (fn [& _] false)]+ (is (false? (check)))))))))++(deftest support-check-caching-test+ (with-redefs [semantic.db.datasource/db-url nil+ mdb/db-type (constantly :postgres)]+ (testing "nothing cached (and mode :unavailable) before the app db is set up"+ (with-support-cache nil+ (with-redefs [mdb/db-is-set-up? (constantly false)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not probe before the app db is set up")))]+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))))+ (testing "check runs exactly once; its result is cached for subsequent calls"+ (with-support-cache nil+ (let [calls (atom 0)]+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (fn [] (swap! calls inc) true)]+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))+ (is (= 1 @calls))+ (is (true? @semantic.db.datasource/app-db-pgvector-support))))))+ (testing "a failed probe reads as unavailable, does NOT latch, and backs off before retrying"+ (with-support-cache nil+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (fn [] (throw (ex-info "boom" {})))]+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))+ (is (nil? @semantic.db.datasource/app-db-pgvector-support)))+ (testing "within the backoff window the check must not run again"+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not re-probe during backoff")))]+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))))+ (testing "once the window clears, the next call re-probes"+ (reset! semantic.db.datasource/probe-cooldown-timer nil)+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))+ ;; This exercises the false->true transition with the probe mocked.+ ;; The real thing (an app-db role reads unsupported, an admin runs CREATE EXTENSION out-of-band, the+ ;; next probe reads installed and the mode flips with no restart) was verified by hand against a real+ ;; Postgres.+ ;; We deliberately don't automate it: it needs a real DB whose extension state changes mid-test, which+ ;; would be flaky against the database shared with other tests between runs.+ (testing "an unsupported probe is NOT latched — it re-probes after the cooldown, so a runtime install is picked up"+ (with-support-cache nil+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (constantly false)]+ (is (= :unavailable (semantic.db.datasource/pgvector-mode)))+ (is (nil? @semantic.db.datasource/app-db-pgvector-support))+ (testing "within the cooldown the check must not run again"+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not re-probe during cooldown")))]+ (is (= :unavailable (semantic.db.datasource/pgvector-mode))))))+ (testing "cooldown elapsed: pgvector installed at runtime is now picked up (no restart)"+ (reset! semantic.db.datasource/probe-cooldown-timer nil)+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))+ (testing "a confirmed true latches for the JVM lifetime; tests/REPL reset it"+ (with-support-cache nil+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support (constantly true)]+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))+ (is (true? @semantic.db.datasource/app-db-pgvector-support)))+ (testing "latched: a now-failing check is never consulted"+ (with-redefs [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not re-probe after a confirmed true")))]+ (is (= :app-db (semantic.db.datasource/pgvector-mode)))))))))++(deftest unlicensed-availability-check-does-not-probe-test+ (testing "without the :semantic-search feature, the availability gate never probes the app db"+ (with-redefs [semantic.db.datasource/db-url nil+ mdb/db-type (constantly :postgres)+ mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/check-app-db-pgvector-support+ (fn [] (throw (AssertionError. "must not probe when the feature is off")))]+ (with-support-cache nil+ (mt/with-premium-features #{}+ (is (false? (semantic.util/semantic-search-available?))))+ (is (nil? @semantic.db.datasource/app-db-pgvector-support))))))++(deftest ensure-initialized-data-source-app-db-test+ (testing "app-db mode hands back the shared application pool without storing it"+ (with-redefs [semantic.db.datasource/db-url nil+ semantic.db.datasource/data-source (atom nil)+ mdb/db-type (constantly :postgres)+ mdb/data-source (constantly ::app-pool)]+ (with-support-cache true+ (is (= ::app-pool (semantic.db.datasource/ensure-initialized-data-source!)))+ (testing "the shared pool is never stored in the module's data-source atom"+ (is (nil? @semantic.db.datasource/data-source)))+ (testing "shutdown-db! therefore cannot close the shared pool"+ ;; would throw on ::app-pool (not a PooledDataSource) if it tried+ (semantic.db.datasource/shutdown-db!)+ (is (= ::app-pool (semantic.db.datasource/ensure-initialized-data-source!))))))))++(deftest ensure-initialized-data-source-unavailable-test+ (testing "no pgvector anywhere → an actionable error"+ (with-redefs [semantic.db.datasource/db-url nil+ semantic.db.datasource/data-source (atom nil)+ mdb/db-type (constantly :h2)]+ (with-support-cache nil+ (is (thrown-with-msg? ExceptionInfo #"MB_PGVECTOR_DB_URL"+ (semantic.db.datasource/ensure-initialized-data-source!)))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.cljindex b3035593fbe..418aba5688d 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj@@ -122,6 +122,38 @@(is (= 2 (:retry_count (first results))))(is (= t2 (:error_gated_at (first results))))))))))+(deftest dlq-entry-upsert-schema-qualified-test+ (testing "the conflict predicate references the target by bare name when the DLQ table is schema-qualified+ (app-db mode); a qualified reference would not resolve inside ON CONFLICT DO UPDATE"+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ schema (str "dlq_test_schema_" (System/nanoTime))+ index-metadata (assoc (semantic.tu/unique-index-metadata)+ :schema schema+ :index-table-qualifier (str schema ".%s"))+ index-id 42]+ (try+ (jdbc/execute! pgvector [(str "CREATE SCHEMA " schema)])+ (semantic.dlq/create-dlq-table-if-not-exists! pgvector index-metadata index-id)+ (let [t1 (ts "2025-01-01T10:00:00Z")+ t2 (ts "2025-01-01T11:00:00Z")+ entry (fn [gated-at retry]+ [{:gate_id "gate1"+ :retry_count retry+ :attempt_at t1+ :last_attempted_at t2+ :error_gated_at gated-at}])]+ (is (= 1 (semantic.dlq/add-entries! pgvector index-metadata index-id (entry t1 0))))+ (is (= 1 (semantic.dlq/add-entries! pgvector index-metadata index-id (entry t2 2)))+ "the conflict-update path must work against a schema-qualified table")+ (let [table-name (semantic.dlq/dlq-table-name-kw index-metadata index-id)+ [row] (jdbc/execute! pgvector+ (sql/format {:select [:*] :from [table-name]} :quoted true)+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (is (= 2 (:retry_count row)))+ (is (= t2 (:error_gated_at row)))))+ (finally+ (jdbc/execute! pgvector [(str "DROP SCHEMA IF EXISTS " schema " CASCADE")]))))))+(deftest dlq-poll-test(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.cljindex 239f788011c..eb29a846d83 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj@@ -17,6 +17,7 @@[metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util.json :as json][toucan2.core :as t2])(:import@@ -25,6 +26,10 @@(set! *warn-on-reflection* true)+;; the token-tracking test hits the app db before any auto-initializing mt helper when it is the first+;; db touch in a fresh JVM+(use-fixtures :once (fixtures/initialize :db))+(deftest test-get-provider(testing "get-active-model returns based on setting"(mt/with-temporary-setting-values [ee-embedding-provider "ai-service"diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 1f57e677627..e52ec127174 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -19,7 +19,8 @@[metabase.permissions.core :as perms][metabase.search.core :as search][metabase.test :as mt]- [metabase.util :as u])+ [metabase.util :as u]+ [next.jdbc :as jdbc])(:import(org.postgresql.util PGobject)))@@ -206,6 +207,20 @@(is (zero? (semantic.tu/index-count index)))(is (=? (expected-index-defs index) (semantic.tu/table-indexes table-name))))))))+(deftest create-index-table!-extension-failure-test+ (testing "a failed CREATE EXTENSION surfaces the actionable pgvector guidance, not the generic wrapper"+ (mt/with-premium-features #{:semantic-search}+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ ;; the extension is the first statement create! runs, so throwing here exercises the inner catch+ (mt/with-dynamic-fn-redefs [jdbc/execute!+ (fn [& _] (throw (RuntimeException. "permission denied to create extension")))]+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (semantic.index/create-index-table-if-not-exists!+ (semantic.env/get-pgvector-datasource!) @index-ref)))]+ (testing "the escaping error keeps the extension-install type and message, not \"Failed to create index table\""+ (is (= ::semantic.index/extension-install-failed (:type (ex-data e))))+ (is (re-find #"pgvector extension" (ex-message e))))))))))+(deftest create-hnsw-index-if-not-exists!-test(mt/with-premium-features #{:semantic-search}(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex 554ae6906af..de40a2b0254 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -11,7 +11,9 @@[metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index :as semantic.index]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.collections.models.collection :as collection][metabase.test :as mt][metabase.util :as u]@@ -34,9 +36,9 @@semantic.db.migration.impl/migrate-dynamic-schema! (constantly nil)](log.capture/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.db.migration :info]](semantic.db.connection/with-migrate-tx [tx]- (semantic.db.migration/maybe-migrate! tx nil)+ (semantic.db.migration/maybe-migrate! tx {:index-metadata semantic.index-metadata/default-index-metadata}){:messages (messages)- :db-version (@#'semantic.db.migration/db-version tx)}))))]+ :db-version (@#'semantic.db.migration/db-version semantic.index-metadata/default-index-metadata tx)}))))](testing "Migration up works"(u/prog1 (migrate-and-get-db-version 130)(is (= 130 (:db-version <>)))@@ -56,6 +58,76 @@(is (m/find-first (comp #{"Migration already performed, skipping."} :message)(:messages <>)))))))))+(deftest schema-scoped-migration-drop-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (testing "app-db-mode migration reset only ever drops tables inside the module's schema"+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata semantic.index-metadata/app-db-index-metadata+ schema-tables (fn []+ (->> (jdbc/execute! pgvector+ ["SELECT tablename FROM pg_tables WHERE schemaname = 'semantic_search'"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (map :tablename)+ set))]+ ;; decoys playing the role of application tables; the second is a real app-db table name+ ;; (migration 056) squarely inside the module's naming family — the worst case for any+ ;; name-pattern-based scoping+ (jdbc/execute! pgvector ["CREATE TABLE decoy_app_table (id int)"])+ (jdbc/execute! pgvector ["CREATE TABLE semantic_search_token_tracking (id int)"])+ ;; positive control: a leftover table inside the module schema is fair game for the reset+ (jdbc/execute! pgvector ["CREATE SCHEMA semantic_search"])+ (jdbc/execute! pgvector ["CREATE TABLE semantic_search.legacy_junk (id int)"])+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))+ (testing "tables outside the schema survive the version<2 drop-everything reset"+ (is (semantic.util/table-exists? pgvector "public.decoy_app_table"))+ (is (semantic.util/table-exists? pgvector "public.semantic_search_token_tracking")))+ (testing "inside the schema: junk dropped, module tables created"+ (is (= #{"migration" "index_metadata" "index_control" "index_gate"}+ (schema-tables)))))))))++(deftest dedicated-reset-stays-in-default-schema-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (testing "the dedicated-mode reset wipes only the default schema — cohabitant schemas survive"+ (let [pgvector (semantic.env/get-pgvector-datasource!)]+ (jdbc/execute! pgvector ["CREATE SCHEMA cohabitant"])+ (jdbc/execute! pgvector ["CREATE TABLE cohabitant.precious (id int)"])+ (jdbc/execute! pgvector ["CREATE TABLE doomed (id int)"])+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate! tx {:index-metadata semantic.index-metadata/default-index-metadata}))+ (is (true? (semantic.util/table-exists? pgvector "cohabitant.precious")))+ (is (false? (semantic.util/table-exists? pgvector "public.doomed"))))))))++(deftest dedicated-reset-refuses-app-db-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (testing "the dedicated-mode reset refuses a database that looks like a Metabase app db"+ (let [pgvector (semantic.env/get-pgvector-datasource!)]+ ;; core_user is the app-db sentinel; a generic Liquibase table alongside it must not mask the refusal+ (jdbc/execute! pgvector ["CREATE TABLE databasechangelog (id int)"])+ (jdbc/execute! pgvector ["CREATE TABLE core_user (id int)"])+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Metabase application"+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate!+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))))+ (testing "nothing was dropped"+ (is (true? (semantic.util/table-exists? pgvector "public.core_user")))))))))++(deftest dedicated-reset-allows-liquibase-only-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (testing "a generic Liquibase databasechangelog does not, on its own, look like a Metabase app db"+ (let [pgvector (semantic.env/get-pgvector-datasource!)]+ (jdbc/execute! pgvector ["CREATE TABLE databasechangelog (id int)"])+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate!+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))+ (testing "migration proceeds: the module tables are created and the stray table is wiped"+ (is (true? (semantic.util/table-exists? pgvector "public.index_metadata")))+ (is (false? (semantic.util/table-exists? pgvector "public.databasechangelog")))))))))+(defn- executions-overlap?"Check if any two executions overlap in time. Each entry is [tid :started/:ended timestamp].Returns true if there's any overlap (which would indicate lock failure)."diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.cljindex f9b62da33a6..4918fa67a81 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj@@ -19,6 +19,17 @@(set! *warn-on-reflection* true)+(deftest ^:parallel schema-scoping-helpers-test+ (testing "scope-where-to-schema adds the schema predicate only in shared app-db mode (non-nil schema)"+ (is (= [:and [:like :t.table_name [:inline "x_%"]]]+ (#'sut/scope-where-to-schema [:and [:like :t.table_name [:inline "x_%"]]] nil)))+ (is (= [:and [:like :t.table_name [:inline "x_%"]] [:= :t.table_schema [:inline "semantic_search"]]]+ (#'sut/scope-where-to-schema [:and [:like :t.table_name [:inline "x_%"]]] "semantic_search"))))+ (testing "requalify-table-names prefixes the schema only in shared app-db mode"+ (is (= ["a" "b"] (#'sut/requalify-table-names nil ["a" "b"])))+ (is (= ["semantic_search.a" "semantic_search.b"]+ (#'sut/requalify-table-names "semantic_search" ["a" "b"])))))+(use-fixtures :once #'semantic.tu/once-fixture)(defn- create-test-index@@ -228,25 +239,25 @@recent-time (t/minus (t/instant) (t/hours 1))] ; 1 hour ago(testing "parse-repair-table-timestamp"(let [repair-table-name (with-redefs [t/instant (constantly old-time)]- (#'repair/repair-table-name))+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))parsed-time (#'sut/parse-repair-table-timestamp repair-table-name)](is (= (t/truncate-to old-time :millis)parsed-time))))(testing "orphan repair table detection and cleanup"(let [old-repair-table-name (with-redefs [t/instant (constantly old-time)]- (#'repair/repair-table-name))+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))recent-repair-table-name (with-redefs [t/instant (constantly recent-time)]- (#'repair/repair-table-name))+ (#'repair/repair-table-name semantic.index-metadata/default-index-metadata))non-repair-table-name "regular_table_123"](try(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" old-repair-table-name)])(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" recent-repair-table-name)])(jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id INT)" non-repair-table-name)])(mt/with-dynamic-fn-redefs [semantic.settings/repair-table-retention-hours (constantly retention-hours)]- (let [orphan-tables (#'sut/orphan-repair-tables pgvector)]+ (let [orphan-tables (#'sut/orphan-repair-tables pgvector semantic.index-metadata/default-index-metadata)](is (= #{old-repair-table-name} (set orphan-tables))"Only old repair table should be detected as orphan"))- (#'sut/cleanup-orphan-repair-tables! pgvector)+ (#'sut/cleanup-orphan-repair-tables! pgvector semantic.index-metadata/default-index-metadata)(is (not (semantic.tu/table-exists-in-db? old-repair-table-name))"Old repair table should be dropped")(is (semantic.tu/table-exists-in-db? recent-repair-table-name)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.cljindex b2b63a40733..f416db3c3bf 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj@@ -4,12 +4,14 @@[honey.sql :as sql][java-time.api :as t][metabase-enterprise.semantic-search.dlq :as semantic.dlq]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.task.metric-collector :as semantic.task.collector][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.u][metabase.test :as mt][next.jdbc :as jdbc]))@@ -71,7 +73,12 @@index-metadata (semantic.tu/unique-index-metadata)model semantic.tu/mock-embedding-model](mt/with-dynamic-fn-redefs [semantic.env/get-index-metadata (fn [] index-metadata)- semantic.env/get-configured-embedding-model (fn [] model)]+ semantic.env/get-configured-embedding-model (fn [] model)+ ;; supported? requires a configured embedder for engine selection+ semantic.embedding/get-configured-model (fn [] model)+ ;; collect-metrics! only runs when semantic is the active engine; pin it so a+ ;; sibling test leaking the search-engine setting can't make this read 0+ semantic.u/semantic-search-active? (fn [] true)](testing "Missing tables are handled gracefully"(let [result (try(@#'semantic.task.collector/collect-metrics!)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.cljindex b2cf2fa2150..c595529ccd7 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer][metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.test.fixtures :as fixtures][toucan2.core :as t2])(:import(java.sql Timestamp)@@ -10,6 +11,10 @@(set! *warn-on-reflection* true)+;; raw t2 access below runs before any auto-initializing mt helper — on the appdb-mode CI job this+;; namespace can be an early db touch in a fresh JVM+(use-fixtures :once (fixtures/initialize :db))+(deftest trim-test(when (semantic.u/semantic-search-available?)(t2/delete! :model/SemanticSearchTokenTracking)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.cljindex 27061cf11ae..f5e8629e22c 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj@@ -52,6 +52,11 @@(defn do-with-temp-datasource!"Impl [[with-temp-datasource]]."[db-name thunk]+ ;; with no URL the redef'd nil db-url could resolve to :app-db mode and hand back the application+ ;; pool — the dedicated harness (which DROPs/CREATEs databases) must never point at the app db.+ ;; unconditional (not assert): this guards destructive setup, and asserts can be elided+ (when (str/blank? (:mb-pgvector-db-url env))+ (throw (ex-info "with-temp-datasource! requires the dedicated-harness MB_PGVECTOR_DB_URL" {})))(with-redefs [semantic.db.datasource/db-url (alt-db-name-url (:mb-pgvector-db-url env) db-name)semantic.db.datasource/data-source (atom nil)](try@@ -559,7 +564,7 @@(semantic.index/drop-index-table! pgvector {:table-name table_name})(semantic.dlq/drop-dlq-table-if-exists! pgvector index-metadata index-id))(semantic.index-metadata/drop-tables-if-exists! pgvector index-metadata)- (semantic.db.migration/drop-migration-table! pgvector))+ (semantic.db.migration/drop-migration-table! index-metadata pgvector))(defn get-table-names [pgvector](->> ["select table_name from information_schema.tables"]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.cljnew file mode 100644index 00000000000..9cff3b055ee--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj@@ -0,0 +1,50 @@+(ns metabase-enterprise.semantic-search.util-test+ (:require+ [clojure.test :refer [deftest is testing use-fixtures]]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]))++(set! *warn-on-reflection* true)++(use-fixtures :once #'semantic.tu/once-fixture)++(deftest ^:parallel quote-ident-test+ (testing "wraps in double quotes and doubles any embedded double quote (the next.jdbc.quoted/postgres contract)"+ (is (= "\"semantic_search\"" (semantic.util/quote-ident "semantic_search")))+ (is (= "\"weird\"\"name\"" (semantic.util/quote-ident "weird\"name")))))++(deftest ^:parallel column-keyword-test+ (testing "a dotted-name keyword (renders as separate identifiers), never a namespaced one"+ (is (= :index_table_1.model (semantic.util/column-keyword "index_table_1" "model")))+ (is (= :semantic_search.index_table_1.model+ (semantic.util/column-keyword "semantic_search.index_table_1" "model")))+ (is (nil? (namespace (semantic.util/column-keyword "semantic_search.t" "c")))))+ (testing "conflict-target-column drops the schema — ON CONFLICT names the target by its bare relation"+ (is (= :index_table_1.model+ (semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))++(deftest catalog-lookups-schema-scoping-test+ (testing "qualified names scope table/index existence checks to their schema; a same-named object in+ another schema (e.g. the app db's public) must not satisfy them"+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (let [pgvector (semantic.env/get-pgvector-datasource!)]+ (jdbc/execute! pgvector ["CREATE TABLE decoy (id int)"])+ (jdbc/execute! pgvector ["CREATE INDEX decoy_idx ON decoy (id)"])+ (jdbc/execute! pgvector ["CREATE SCHEMA s2"])+ (testing "unqualified names keep the historical any-schema behavior"+ (is (true? (semantic.util/table-exists? pgvector "decoy")))+ (is (true? (semantic.util/index-exists? pgvector "decoy_idx"))))+ (testing "qualified names are not fooled by the public decoys"+ (is (true? (semantic.util/table-exists? pgvector "public.decoy")))+ (is (true? (semantic.util/index-exists? pgvector "public.decoy_idx")))+ (is (false? (semantic.util/table-exists? pgvector "s2.decoy")))+ (is (false? (semantic.util/index-exists? pgvector "s2.decoy_idx"))))+ (testing "qualified names find the objects in their own schema"+ (jdbc/execute! pgvector ["CREATE TABLE s2.decoy (id int)"])+ (jdbc/execute! pgvector ["CREATE INDEX decoy_idx ON s2.decoy (id)"])+ (is (true? (semantic.util/table-exists? pgvector "s2.decoy")))+ (is (true? (semantic.util/index-exists? pgvector "s2.decoy_idx")))))))))