The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Adopt bare library index tables in dedicated store only (#79986)
Chris Truter548bae698605
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.cljindex 47ff16af8fc..022c2c85bcb 100644--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj@@ -23,6 +23,7 @@[clojure.string :as str][honey.sql :as sql][honey.sql.helpers :as sql.helpers]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.util :as semantic.util][metabase.util.log :as log][next.jdbc :as jdbc]@@ -36,7 +37,9 @@(def index-schema"Schema holding the library entity index, in a dedicated store and on the app db alike.Its own, not semantic search's: [[metabase-enterprise.semantic-search.db.migration.impl]] wipes the- schema it is given, and in a dedicated store that is the default schema these tables used to sit in."+ schema it is given, and a dedicated store's reset sweeps the default schema these tables used to sit in.+ It spares the names it recognizes, and ours were never among them -- but an index should not depend on+ another module's list of what it may destroy.""library_retrieval");; TODO (Chris 2026-07-14) -- these names carry no version, so an on-disk upgrade has nothing to key off.@@ -192,12 +195,18 @@{:type ::schema-creation-failed, :schema schema}e))))))+;; TODO (Chris 2026-08-17) -- delete this once no supported upgrade can start from the bare names.(defn- adopt-legacy-tables!"Move an index built before [[index-schema]] existed into it, preserving its rows.Without this, the qualified tables would be created empty beside the old ones and the whole library- would be re-embedded. A no-op once moved: the unqualified name no longer resolves."+ would be re-embedded. A no-op once moved: the unqualified name no longer resolves.++ A dedicated store only. The bare names date from when this index was dedicated-only, so nothing in an+ app db can be an old index of ours -- but [[table-exists?]] resolves an unqualified name on the search+ path, where an application table that happens to share the name would answer, and get moved."[tx]- (when (= default-tables (tables))+ (when (and (= default-tables (tables))+ (semantic.db.datasource/dedicated-url-configured?))(doseq [k [:vectors :meta]](let [legacy (k legacy-tables)](when (and (table-exists? tx legacy)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex 284902e3019..810e3f12b66 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -53,9 +53,10 @@When index-metadata carries a `:schema` (shared app-db mode) ONLY tables inside that schema may bedropped — the application's tables live in other schemas and must never be touched here.Without a `:schema` the database is assumed dedicated to semantic search and its default schema is- swept, but only for tables [[semantic-search-table?]] recognizes — library retrieval keeps its index- there too. Refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed- at the application database would otherwise destroy it here, on first init)."+ swept, but only for tables [[semantic-search-table?]] recognizes — the schema is shared, and an index+ library retrieval built before it had a schema of its own still sits there until adoption moves it.+ Refuses outright when the database looks like a Metabase app db (MB_PGVECTOR_DB_URL pointed at the+ application database would otherwise destroy it here, on first init)."[index-metadata tx](let [schema (:schema index-metadata)tables (jdbc/execute! txdiff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.cljindex b098bef3d6d..a12d09b95b2 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/appdb_mode_test.clj@@ -71,6 +71,10 @@(testing "opted in, but a library_retrieval schema already exists — refusing to clobber it"(is false "the appdb-mode job should start from a fresh app db"))+ (contains? (tables-in-schema (mdb/data-source) "public") (:vectors index-table/legacy-tables))+ (testing "opted in, but a public library_entity_index already exists — refusing to clobber it"+ (is false "the round trip creates that table itself; drop the leftover from an interrupted run"))+:else(mt/with-premium-features #{:library :library-retrieval};; near-identical vectors, so the query lands on its document and nothing else@@ -91,19 +95,14 @@(is (= :app-db (semantic.db.datasource/pgvector-mode)))(is (true? (entity-retrieval.core/available?)))(is (= index-table/default-tables (index-table/tables))))+ ;; an application table carrying the name this index used before it had a schema of its own.+ ;; No app db can hold an old index of ours -- the bare names date from when the index was+ ;; dedicated-only -- so nothing may move this one, whatever it is called.+ (jdbc/execute! app-db ["CREATE TABLE library_entity_index (id bigint primary key)"])+ (jdbc/execute! app-db ["INSERT INTO library_entity_index VALUES (1)"])(let [ds (semantic.db.datasource/ensure-initialized-data-source!)public-before (tables-in-schema app-db "public")](is (identical? app-db ds) "app-db mode shares the application pool")- ;; an index built before the schema existed, standing in for an upgrading instance. Its- ;; rows must survive: rebuilding instead would re-embed the whole library.- ;; ensure-tables! installs the extension, but this table predates that call and its- ;; vector(4) column needs the type to exist already- (jdbc/execute! app-db ["CREATE EXTENSION IF NOT EXISTS vector"])- (jdbc/execute! app-db ["CREATE TABLE library_entity_index- (doc_id text primary key, entity_type text, entity_local_id bigint,- doc_type text, doc_text text, doc_embedding vector(4))"])- (jdbc/execute! app-db ["INSERT INTO library_entity_index- VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}:model/Collection {data-id :id} {:type "library-data":location (str "/" lib-id "/")}@@ -114,22 +113,17 @@:active true:name "dog_training_guide":display_name "Dog Training Guide"}]- (testing "adoption keeps the rows an upgrading instance already embedded"- ;; asserted here rather than after the reconcile, which GCs any document the library- ;; no longer implies -- including this synthetic one- (index-table/ensure-tables! ds semantic.tu/mock-embedding-model)+ (index-table/ensure-tables! ds semantic.tu/mock-embedding-model)+ (testing "an application table sharing the index's old bare name is left where it is"(is (= 1 (:count (jdbc/execute-one!app-db- [(format "SELECT count(*) AS count FROM %s WHERE doc_id = 'legacy-doc'"- (index-table/vectors-table-sql))]+ ["SELECT count(*) AS count FROM public.library_entity_index"]{:builder-fn jdbc.rs/as-unqualified-lower-maps})))- "moved, not recreated empty -- otherwise the whole library re-embeds"))+ "adoption belongs to a dedicated store, whose bare tables really are ours"))(reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model))(testing "both tables live inside the library_retrieval schema"(is (= #{"library_entity_index" "library_entity_index_meta"}(tables-in-schema app-db index-table/index-schema))))- (testing "the pre-existing index was moved, not left behind"- (is (not (contains? (tables-in-schema app-db "public") "library_entity_index"))))(testing "the application schema is untouched"(is (= public-before (tables-in-schema app-db "public"))))(testing "retrieval round-trips through the app-db pool"@@ -148,5 +142,6 @@(is (= #{"library_entity_index" "library_entity_index_meta"}(tables-in-schema app-db index-table/index-schema))))))(finally+ (jdbc/execute! app-db ["DROP TABLE IF EXISTS public.library_entity_index"])(jdbc/execute! app-db [(format "DROP SCHEMA IF EXISTS %s CASCADE"index-table/index-schema)])))))))))diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.cljindex 5d7e096e210..a15415a005c 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj@@ -3,9 +3,12 @@[clojure.test :refer :all][metabase-enterprise.entity-retrieval.index-table :as index-table][metabase-enterprise.entity-retrieval.reconcile :as reconcile]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase.test :as mt][metabase.util.log.capture :as log.capture]- [next.jdbc :as jdbc]))+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]))(deftest legacy-meta-table-without-ownership-test(testing "a grant-only role neither attempts ALTER nor repeatedly warns"@@ -60,8 +63,9 @@(deftest legacy-tables-are-moved-not-rebuilt-test(let [statements (atom [])exists (atom #{"library_entity_index" "library_entity_index_meta"})]- (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))- #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)}+ (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))+ #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)+ #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}(fn [](testing "an index built before the schema existed is moved, keeping its rows"(#'index-table/adopt-legacy-tables! ::tx)@@ -82,7 +86,49 @@"library_retrieval.library_entity_index_meta"})(#'index-table/adopt-legacy-tables! ::tx)(is (empty? @statements)- "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))))))+ "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))+ (testing "an app db is never adopted from: nothing there can be an old index of ours"+ (reset! statements [])+ (reset! exists #{"library_entity_index" "library_entity_index_meta"})+ (with-redefs [semantic.db.datasource/db-url nil]+ (#'index-table/adopt-legacy-tables! ::tx))+ (is (empty? @statements)+ "an application table on the search path answers to the bare name, and must not be moved"))))))++(deftest ^:synchronized legacy-tables-are-adopted-in-a-dedicated-store-test+ (testing "a real SET SCHEMA carries the rows an upgrading dedicated store already embedded"+ (when (semantic.db.datasource/dedicated-url-configured?)+ ;; its own database: adoption only runs under the real table names, which a shared store may hold+ (semantic.tu/with-test-db! {:dbname "library_retrieval_adoption_test" :mode :blank :cleanup :both}+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)]+ (jdbc/execute! pgvector ["CREATE EXTENSION IF NOT EXISTS vector"])+ ;; both bare names the index carried before it had a schema of its own, with a meta row for the+ ;; model it was built against -- an upgrade the reconciler should find nothing to redo+ (jdbc/execute! pgvector ["CREATE TABLE library_entity_index+ (doc_id text primary key, entity_type text, entity_local_id bigint,+ doc_type text, doc_text text, doc_embedding vector(4))"])+ (jdbc/execute! pgvector ["INSERT INTO library_entity_index+ VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])+ (jdbc/execute! pgvector ["CREATE TABLE library_entity_index_meta+ (id smallint primary key, provider text, model_name text,+ vector_dimensions int, schema_version int,+ updated_at timestamptz, reconciled_at timestamptz)"])+ (jdbc/execute! pgvector ["INSERT INTO library_entity_index_meta+ VALUES (1, ?, ?, ?, ?, now(), now())"+ (:provider semantic.tu/mock-embedding-model)+ (:model-name semantic.tu/mock-embedding-model)+ (:vector-dimensions semantic.tu/mock-embedding-model)+ index-table/schema-version])+ (is (= :ok (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model))+ "the adopted meta row still describes the configured model, so nothing is rebuilt")+ (is (= ["legacy-doc"]+ (map :doc_id (jdbc/execute! pgvector+ [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))+ "moved, not recreated empty -- otherwise the whole library re-embeds on upgrade")+ (is (empty? (jdbc/execute! pgvector [(str "SELECT tablename FROM pg_tables WHERE schemaname = 'public'"+ " AND tablename LIKE 'library_entity_index%'")]))+ "moved, not copied -- a leftover is a stale duplicate of the whole index"))))))(deftest reconcile-watermark-precedes-appdb-read-test(let [events (atom [])]