Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Adopt bare library index tables in dedicated store only (#79986)

Chris Truter548bae698605
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
index 47ff16af8fc..022c2c85bcb 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
@@ -23,6 +23,7 @@
[clojure.string :as str]
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -36,7 +37,9 @@
(def index-schema
"Schema holding the library entity index, in a dedicated store and on the app db alike.
Its own, not semantic search's: [[metabase-enterprise.semantic-search.db.migration.impl]] wipes the
- schema it is given, and in a dedicated store that is the default schema these tables used to sit in."
+ schema it is given, and a dedicated store's reset sweeps the default schema these tables used to sit in.
+ It spares the names it recognizes, and ours were never among them -- but an index should not depend on
+ another module's list of what it may destroy."
"library_retrieval")
;; TODO (Chris 2026-07-14) -- these names carry no version, so an on-disk upgrade has nothing to key off.
@@ -192,12 +195,18 @@
{:type ::schema-creation-failed, :schema schema}
e))))))
+;; TODO (Chris 2026-08-17) -- delete this once no supported upgrade can start from the bare names.
(defn- adopt-legacy-tables!
"Move an index built before [[index-schema]] existed into it, preserving its rows.
Without this, the qualified tables would be created empty beside the old ones and the whole library
- would be re-embedded. A no-op once moved: the unqualified name no longer resolves."
+ would be re-embedded. A no-op once moved: the unqualified name no longer resolves.
+
+ A dedicated store only. The bare names date from when this index was dedicated-only, so nothing in an
+ app db can be an old index of ours -- but [[table-exists?]] resolves an unqualified name on the search
+ path, where an application table that happens to share the name would answer, and get moved."
[tx]
- (when (= default-tables (tables))
+ (when (and (= default-tables (tables))
+ (semantic.db.datasource/dedicated-url-configured?))
(doseq [k [:vectors :meta]]
(let [legacy (k legacy-tables)]
(when (and (table-exists? tx legacy)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index 284902e3019..810e3f12b66 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -53,9 +53,10 @@
When index-metadata carries a `:schema` (shared app-db mode) ONLY tables inside that schema may be
dropped — the application's tables live in other schemas and must never be touched here.
Without a `:schema` the database is assumed dedicated to semantic search and its default schema is
- swept, but only for tables [[semantic-search-table?]] recognizes — library retrieval keeps its index
- there too. Refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed
- at the application database would otherwise destroy it here, on first init)."
+ swept, but only for tables [[semantic-search-table?]] recognizes — the schema is shared, and an index
+ library retrieval built before it had a schema of its own still sits there until adoption moves it.
+ Refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed at the
+ application database would otherwise destroy it here, on first init)."
[index-metadata tx]
(let [schema (:schema index-metadata)
tables (jdbc/execute! tx
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj
index b098bef3d6d..a12d09b95b2 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj
@@ -71,6 +71,10 @@
(testing "opted in, but a library_retrieval schema already exists — refusing to clobber it"
(is false "the appdb-mode job should start from a fresh app db"))
+ (contains? (tables-in-schema (mdb/data-source) "public") (:vectors index-table/legacy-tables))
+ (testing "opted in, but a public library_entity_index already exists — refusing to clobber it"
+ (is false "the round trip creates that table itself; drop the leftover from an interrupted run"))
+
:else
(mt/with-premium-features #{:library :library-retrieval}
;; near-identical vectors, so the query lands on its document and nothing else
@@ -91,19 +95,14 @@
(is (= :app-db (semantic.db.datasource/pgvector-mode)))
(is (true? (entity-retrieval.core/available?)))
(is (= index-table/default-tables (index-table/tables))))
+ ;; an application table carrying the name this index used before it had a schema of its own.
+ ;; No app db can hold an old index of ours -- the bare names date from when the index was
+ ;; dedicated-only -- so nothing may move this one, whatever it is called.
+ (jdbc/execute! app-db ["CREATE TABLE library_entity_index (id bigint primary key)"])
+ (jdbc/execute! app-db ["INSERT INTO library_entity_index VALUES (1)"])
(let [ds (semantic.db.datasource/ensure-initialized-data-source!)
public-before (tables-in-schema app-db "public")]
(is (identical? app-db ds) "app-db mode shares the application pool")
- ;; an index built before the schema existed, standing in for an upgrading instance. Its
- ;; rows must survive: rebuilding instead would re-embed the whole library.
- ;; ensure-tables! installs the extension, but this table predates that call and its
- ;; vector(4) column needs the type to exist already
- (jdbc/execute! app-db ["CREATE EXTENSION IF NOT EXISTS vector"])
- (jdbc/execute! app-db ["CREATE TABLE library_entity_index
- (doc_id text primary key, entity_type text, entity_local_id bigint,
- doc_type text, doc_text text, doc_embedding vector(4))"])
- (jdbc/execute! app-db ["INSERT INTO library_entity_index
- VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])
(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}
:model/Collection {data-id :id} {:type "library-data"
:location (str "/" lib-id "/")}
@@ -114,22 +113,17 @@
:active true
:name "dog_training_guide"
:display_name "Dog Training Guide"}]
- (testing "adoption keeps the rows an upgrading instance already embedded"
- ;; asserted here rather than after the reconcile, which GCs any document the library
- ;; no longer implies -- including this synthetic one
- (index-table/ensure-tables! ds semantic.tu/mock-embedding-model)
+ (index-table/ensure-tables! ds semantic.tu/mock-embedding-model)
+ (testing "an application table sharing the index's old bare name is left where it is"
(is (= 1 (:count (jdbc/execute-one!
app-db
- [(format "SELECT count(*) AS count FROM %s WHERE doc_id = 'legacy-doc'"
- (index-table/vectors-table-sql))]
+ ["SELECT count(*) AS count FROM public.library_entity_index"]
{:builder-fn jdbc.rs/as-unqualified-lower-maps})))
- "moved, not recreated empty -- otherwise the whole library re-embeds"))
+ "adoption belongs to a dedicated store, whose bare tables really are ours"))
(reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model))
(testing "both tables live inside the library_retrieval schema"
(is (= #{"library_entity_index" "library_entity_index_meta"}
(tables-in-schema app-db index-table/index-schema))))
- (testing "the pre-existing index was moved, not left behind"
- (is (not (contains? (tables-in-schema app-db "public") "library_entity_index"))))
(testing "the application schema is untouched"
(is (= public-before (tables-in-schema app-db "public"))))
(testing "retrieval round-trips through the app-db pool"
@@ -148,5 +142,6 @@
(is (= #{"library_entity_index" "library_entity_index_meta"}
(tables-in-schema app-db index-table/index-schema))))))
(finally
+ (jdbc/execute! app-db ["DROP TABLE IF EXISTS public.library_entity_index"])
(jdbc/execute! app-db [(format "DROP SCHEMA IF EXISTS %s CASCADE"
index-table/index-schema)])))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
index 5d7e096e210..a15415a005c 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
@@ -3,9 +3,12 @@
[clojure.test :refer :all]
[metabase-enterprise.entity-retrieval.index-table :as index-table]
[metabase-enterprise.entity-retrieval.reconcile :as reconcile]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase.test :as mt]
[metabase.util.log.capture :as log.capture]
- [next.jdbc :as jdbc]))
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
(deftest legacy-meta-table-without-ownership-test
(testing "a grant-only role neither attempts ALTER nor repeatedly warns"
@@ -60,8 +63,9 @@
(deftest legacy-tables-are-moved-not-rebuilt-test
(let [statements (atom [])
exists (atom #{"library_entity_index" "library_entity_index_meta"})]
- (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))
- #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)}
+ (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))
+ #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)
+ #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}
(fn []
(testing "an index built before the schema existed is moved, keeping its rows"
(#'index-table/adopt-legacy-tables! ::tx)
@@ -82,7 +86,49 @@
"library_retrieval.library_entity_index_meta"})
(#'index-table/adopt-legacy-tables! ::tx)
(is (empty? @statements)
- "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))))))
+ "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))
+ (testing "an app db is never adopted from: nothing there can be an old index of ours"
+ (reset! statements [])
+ (reset! exists #{"library_entity_index" "library_entity_index_meta"})
+ (with-redefs [semantic.db.datasource/db-url nil]
+ (#'index-table/adopt-legacy-tables! ::tx))
+ (is (empty? @statements)
+ "an application table on the search path answers to the bare name, and must not be moved"))))))
+
+(deftest ^:synchronized legacy-tables-are-adopted-in-a-dedicated-store-test
+ (testing "a real SET SCHEMA carries the rows an upgrading dedicated store already embedded"
+ (when (semantic.db.datasource/dedicated-url-configured?)
+ ;; its own database: adoption only runs under the real table names, which a shared store may hold
+ (semantic.tu/with-test-db! {:dbname "library_retrieval_adoption_test" :mode :blank :cleanup :both}
+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)]
+ (jdbc/execute! pgvector ["CREATE EXTENSION IF NOT EXISTS vector"])
+ ;; both bare names the index carried before it had a schema of its own, with a meta row for the
+ ;; model it was built against -- an upgrade the reconciler should find nothing to redo
+ (jdbc/execute! pgvector ["CREATE TABLE library_entity_index
+ (doc_id text primary key, entity_type text, entity_local_id bigint,
+ doc_type text, doc_text text, doc_embedding vector(4))"])
+ (jdbc/execute! pgvector ["INSERT INTO library_entity_index
+ VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])
+ (jdbc/execute! pgvector ["CREATE TABLE library_entity_index_meta
+ (id smallint primary key, provider text, model_name text,
+ vector_dimensions int, schema_version int,
+ updated_at timestamptz, reconciled_at timestamptz)"])
+ (jdbc/execute! pgvector ["INSERT INTO library_entity_index_meta
+ VALUES (1, ?, ?, ?, ?, now(), now())"
+ (:provider semantic.tu/mock-embedding-model)
+ (:model-name semantic.tu/mock-embedding-model)
+ (:vector-dimensions semantic.tu/mock-embedding-model)
+ index-table/schema-version])
+ (is (= :ok (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model))
+ "the adopted meta row still describes the configured model, so nothing is rebuilt")
+ (is (= ["legacy-doc"]
+ (map :doc_id (jdbc/execute! pgvector
+ [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+ "moved, not recreated empty -- otherwise the whole library re-embeds on upgrade")
+ (is (empty? (jdbc/execute! pgvector [(str "SELECT tablename FROM pg_tables WHERE schemaname = 'public'"
+ " AND tablename LIKE 'library_entity_index%'")]))
+ "moved, not copied -- a leftover is a stale duplicate of the whole index"))))))
(deftest reconcile-watermark-precedes-appdb-read-test
(let [events (atom [])]