Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Exclude incompatible vector spaces from semantic search https://github.com/siyuan-note/siyuan/issues/20991

Danielced7d6c8166b
diff --git a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20230304000547-ibldj1z/20260804171005-semnt03.sy b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20230304000547-ibldj1z/20260804171005-semnt03.sy
index 5c1cb9db76..60b35a1e9c 100644
--- a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20230304000547-ibldj1z/20260804171005-semnt03.sy
+++ b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20230304000547-ibldj1z/20260804171005-semnt03.sy
@@ -6,7 +6,7 @@
"id": "20260804171005-semnt03",
"title": "Semantic Search",
"type": "doc",
- "updated": "20260809232000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -44,7 +44,7 @@
"HeadingLevel": 2,
"Properties": {
"id": "20260804170126-6x70vu9",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -59,7 +59,7 @@
"HeadingLevel": 3,
"Properties": {
"id": "20260804170127-3fpyace",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -123,12 +123,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20260804170129-j6gqedu",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
"Type": "NodeText",
- "Data": "Set vector dimensions to 0 to use the model default. Specify another value only when the target model supports custom dimensions. Rebuild the embedding index after changing the model or dimensions."
+ "Data": "Set vector dimensions to 0 to use the model default. Specify another value only when the target model supports custom dimensions. Search uses only vectors with the same service address, model and configured dimensions as the query, and checks their actual dimensions. Incompatible vectors and older vectors whose source cannot be verified are retained but excluded from search; changing settings or upgrading does not automatically rebuild them. Use the rebuild embedding index action below to recalculate them when needed; this sends model requests and may incur charges."
}
]
},
diff --git a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20230303235619-ex5l63e/20260804171001-semnt01.sy b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20230303235619-ex5l63e/20260804171001-semnt01.sy
index 220da1cb2a..63467579a1 100644
--- a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20230303235619-ex5l63e/20260804171001-semnt01.sy
+++ b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20230303235619-ex5l63e/20260804171001-semnt01.sy
@@ -6,7 +6,7 @@
"id": "20260804171001-semnt01",
"title": "语义搜索",
"type": "doc",
- "updated": "20260820190627"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -44,7 +44,7 @@
"HeadingLevel": 2,
"Properties": {
"id": "20260804170126-k37rp2l",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -59,7 +59,7 @@
"HeadingLevel": 3,
"Properties": {
"id": "20260804170127-mgcq0zk",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -123,12 +123,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20260804170129-7wv1ug4",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
"Type": "NodeText",
- "Data": "向量维度设为 0 时使用模型默认值。仅在目标模型支持自定义向量维度时填写其他值;修改模型或向量维度后需要重建嵌入索引。"
+ "Data": "向量维度设为 0 时使用模型默认值。仅在目标模型支持自定义向量维度时填写其他值。语义搜索只使用与查询的服务地址、模型及配置维度一致且实际维度匹配的向量。不兼容或无法确认来源的旧向量会保留,但不参与检索;修改配置或升级不会自动重建。需要时可使用下方的重建嵌入索引操作重新计算,这会请求模型服务并可能产生费用。"
}
]
},
diff --git a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20230304000829-9jwu3po/20260804171003-semnt02.sy b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20230304000829-9jwu3po/20260804171003-semnt02.sy
index 42c22c9538..a110e29745 100644
--- a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20230304000829-9jwu3po/20260804171003-semnt02.sy
+++ b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20230304000829-9jwu3po/20260804171003-semnt02.sy
@@ -6,7 +6,7 @@
"id": "20260804171003-semnt02",
"title": "語意搜尋",
"type": "doc",
- "updated": "20260809232000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -44,7 +44,7 @@
"HeadingLevel": 2,
"Properties": {
"id": "20260804170126-8749zd5",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -59,7 +59,7 @@
"HeadingLevel": 3,
"Properties": {
"id": "20260804170127-a8c9zop",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -123,12 +123,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20260804170129-7fit0vr",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
"Type": "NodeText",
- "Data": "向量維度設為 0 時使用模型預設值。僅在目標模型支援自訂向量維度時填寫其他值;修改模型或向量維度後需要重建嵌入索引。"
+ "Data": "向量維度設為 0 時使用模型預設值。僅在目標模型支援自訂向量維度時填寫其他值。語意搜尋只使用與查詢的服務位址、模型及設定維度一致且實際維度相符的向量。不相容或無法確認來源的舊向量會保留,但不參與檢索;修改設定或升級不會自動重建。需要時可使用下方的重建嵌入索引操作重新計算,這會請求模型服務並可能產生費用。"
}
]
},
diff --git a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-ei5t6tt/20260804171007-semnt04.sy b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-ei5t6tt/20260804171007-semnt04.sy
index d5c26795cf..d226e1dd07 100644
--- a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-ei5t6tt/20260804171007-semnt04.sy
+++ b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-ei5t6tt/20260804171007-semnt04.sy
@@ -6,7 +6,7 @@
"id": "20260804171007-semnt04",
"title": "意味検索",
"type": "doc",
- "updated": "20260809232000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -44,7 +44,7 @@
"HeadingLevel": 2,
"Properties": {
"id": "20260804170126-emdywyo",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -59,7 +59,7 @@
"HeadingLevel": 3,
"Properties": {
"id": "20260804170127-50f1cex",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
@@ -119,12 +119,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20260804170129-zepo081",
- "updated": "20260804180000"
+ "updated": "20261027103144"
},
"Children": [
{
"Type": "NodeText",
- "Data": "ベクトル次元を 0 にするとモデルの既定値を使用します。対象モデルがカスタム次元をサポートしている場合のみ別の値を指定してください。モデルまたは次元を変更した後は埋め込みインデックスを再構築します。"
+ "Data": "ベクトル次元を 0 にするとモデルの既定値を使用します。対象モデルがカスタム次元をサポートしている場合のみ別の値を指定してください。検索ではサービスのアドレス、モデル、設定した次元がクエリと一致し、実際の次元も一致するベクトルだけを使用します。互換性がないベクトルや生成元を確認できない旧ベクトルは保持されますが、検索から除外されます。設定変更やアップグレードでは自動的に再構築しません。必要に応じて以下の埋め込みインデックスの再構築を実行してください。モデルへのリクエストにより料金が発生する場合があります。"
}
]
},
diff --git a/kernel/model/embedding.go b/kernel/model/embedding.go
index 0658735a26..7c0b9e6154 100644
--- a/kernel/model/embedding.go
+++ b/kernel/model/embedding.go
@@ -18,6 +18,7 @@ package model
import (
"container/heap"
+ "crypto/sha256"
"encoding/binary"
"fmt"
"math"
@@ -369,7 +370,7 @@ func doEmbedAndStore(texts []string, blocks []map[string]any, generation uint64)
buf := encodeVector(vectors[i])
// 原内容仍存在时才写入,路径取当前值,避免编辑、移动或删除期间回写过期结果。
- err = sql.Exec(stmtStoreEmbedding, buf, model, len(plainText), id, updated, plainText)
+ err = sql.Exec(stmtStoreEmbedding, buf, model, embeddingSpaceID(baseURL, model, dimensions), len(plainText), id, updated, plainText)
if err != nil {
embeddingStop.Store(true)
logging.LogErrorf("store embedding failed for block [%s]: %s", id, err)
@@ -477,12 +478,17 @@ func SemanticSearchBlock(query string, boxes, paths []string, types, subTypes ma
return
}
- vectors, err := util.BatchGetEmbeddings([]string{query}, embeddingKey(), embeddingBaseURL(), embeddingModel(), embeddingDimensions(), embeddingTimeout())
+ key, baseURL, model, dimensions := embeddingKey(), embeddingBaseURL(), embeddingModel(), embeddingDimensions()
+ space := embeddingSpaceID(baseURL, model, dimensions)
+ vectors, err := util.BatchGetEmbeddings([]string{query}, key, baseURL, model, dimensions, embeddingTimeout())
if err != nil || 1 > len(vectors) {
logging.LogErrorf("get query embedding failed")
return
}
queryVec := vectors[0]
+ if len(queryVec) == 0 || space != embeddingSpaceID(embeddingBaseURL(), embeddingModel(), embeddingDimensions()) || !isEmbeddingEnabled() {
+ return
+ }
boxFilter, boxArgs := buildBoxesFilter(boxes, "be.")
pathFilter, pathArgs := buildPathsFilter(paths, "be.")
@@ -507,19 +513,21 @@ func SemanticSearchBlock(query string, boxes, paths []string, types, subTypes ma
for {
var q string
- var args []any
+ args := []any{space, len(queryVec) * embeddingVectorDim}
if hasFilter {
- q = fmt.Sprintf("SELECT be.rowid, be.id, be.embedding FROM block_embeddings be JOIN blocks b ON be.id = b.id WHERE be.embedding IS NOT NULL AND length(be.embedding) > 0 AND be.rowid > %d", cursor)
+ q = fmt.Sprintf("SELECT be.rowid, be.id, be.embedding FROM block_embeddings be JOIN blocks b ON be.id = b.id WHERE be.space = ? AND length(be.embedding) = ? AND be.rowid > %d", cursor)
if hasTypeFilter {
q += " AND " + typeFilter
}
q += boxFilter + pathFilter + boxDocFilter + excludeFilter
// 过滤值通过绑定参数传递,避免 SQL 拼接注入
- args = append(append(append([]any{}, boxArgs...), pathArgs...), boxDocArgs...)
+ args = append(args, boxArgs...)
+ args = append(args, pathArgs...)
+ args = append(args, boxDocArgs...)
args = append(args, excludeArgs...)
q += fmt.Sprintf(" ORDER BY be.rowid LIMIT %d", scanSize)
} else {
- q = fmt.Sprintf("SELECT rowid, id, embedding FROM block_embeddings WHERE embedding IS NOT NULL AND length(embedding) > 0 AND rowid > %d ORDER BY rowid LIMIT %d", cursor, scanSize)
+ q = fmt.Sprintf("SELECT rowid, id, embedding FROM block_embeddings WHERE space = ? AND length(embedding) = ? AND rowid > %d ORDER BY rowid LIMIT %d", cursor, scanSize)
}
rows, qErr := sql.QueryNoLimitArgs(q, args...)
if qErr != nil {
@@ -581,6 +589,10 @@ func SemanticSearchBlock(query string, boxes, paths []string, types, subTypes ma
}
}
+ // Discard the search if settings changed while scanning candidates.
+ if space != embeddingSpaceID(embeddingBaseURL(), embeddingModel(), embeddingDimensions()) || !isEmbeddingEnabled() {
+ return
+ }
matchedBlockCount = h.Len()
if 1 > matchedBlockCount {
pageCount = 0
@@ -866,6 +878,12 @@ func embeddingDimensions() int {
return 0
}
+// Versioned identity of the configured vector space, excluding credentials.
+// The stored byte length additionally verifies the provider's actual output dimension.
+func embeddingSpaceID(baseURL, model string, dimensions int) string {
+ return fmt.Sprintf("v1:%x", sha256.Sum256([]byte(fmt.Sprintf("%q:%q:%d", baseURL, model, dimensions))))
+}
+
func embeddingModel() string {
confAISnapshot := Conf.LoadAI()
diff --git a/kernel/model/embedding_space_fts5_test.go b/kernel/model/embedding_space_fts5_test.go
new file mode 100644
index 0000000000..463a9f4897
--- /dev/null
+++ b/kernel/model/embedding_space_fts5_test.go
@@ -0,0 +1,112 @@
+//go:build fts5
+
+package model
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/conf"
+ "github.com/siyuan-note/siyuan/kernel/sql"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestEmbeddingSpaceSearch(t *testing.T) {
+ const childEnv = "SIYUAN_TEST_EMBEDDING_SPACE"
+ if os.Getenv(childEnv) == "" {
+ cmd := exec.Command(os.Args[0], "-test.run=^TestEmbeddingSpaceSearch$", "-test.timeout=60s", "-test.v")
+ cmd.Env = append(os.Environ(), childEnv+"=1")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("%v\n%s", err, output)
+ }
+ return
+ }
+ setupFileOperationTest(t)
+ Conf.Search.Store(conf.NewSearch())
+ Conf.AI.Store(conf.NewAI())
+ Conf.AI.Load().Embedding = &conf.Embedding{Enabled: true, APIKey: "test", Name: "model", Dimensions: 2}
+ util.TempDir = t.TempDir()
+ util.QueueDir = filepath.Join(util.TempDir, "queue")
+ util.DBPath = filepath.Join(util.TempDir, "siyuan.db")
+ util.HistoryDBPath = filepath.Join(util.TempDir, "history.db")
+ util.AssetContentDBPath = filepath.Join(util.TempDir, "asset_content.db")
+ sql.InitDatabase(true)
+ sql.InitHistoryDatabase(true)
+ if err := sql.InitAssetContentDatabase(true); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(sql.CloseDatabase)
+ embeddingTableOk = true
+ var calls atomic.Int32
+ var switchConfig atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ if switchConfig.Load() {
+ Conf.AI.Load().Embedding.Name = "changed-during-query"
+ }
+ w.Header().Set("Content-Type", "application/json")
+ fmt.Fprint(w, `{"data":[{"embedding":[1,0],"index":0}],"model":"model"}`)
+ }))
+ defer server.Close()
+ Conf.AI.Load().Embedding.BaseURL = server.URL
+ space := embeddingSpaceID(server.URL, "model", 2)
+ exec := func(q string, args ...any) {
+ t.Helper()
+ if err := sql.Exec(q, args...); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for i, item := range []struct {
+ space string
+ vector []float32
+ }{
+ {space, []float32{1, 0}},
+ {embeddingSpaceID(server.URL, "other", 2), []float32{1, 0}},
+ {embeddingSpaceID(server.URL+"/other", "model", 2), []float32{1, 0}},
+ {embeddingSpaceID(server.URL, "model", 3), []float32{1, 0}},
+ {"", []float32{1, 0}},
+ {space, []float32{1, 0, 0}},
+ } {
+ id := fmt.Sprintf("2026102710000%d-test%03d", i, i)
+ exec("INSERT INTO blocks VALUES (?, '', ?, '', 'box', '/doc.sy', '/Doc', '', '', '', '', 'search content', '', 'search content', 14, 'p', '', '', 0, '1', '1')", id, id)
+ exec("INSERT INTO block_embeddings (id, root_id, box, path, embedding, model, space) VALUES (?, ?, 'box', '/doc.sy', ?, 'model', ?)", id, id, encodeVector(item.vector), item.space)
+ }
+ for _, boxes := range [][]string{nil, {"box"}} {
+ blocks, count, _, _ := SemanticSearchBlock("query", boxes, nil, nil, nil, 1, 20, nil, nil)
+ if count != 1 || len(blocks) != 1 || blocks[0].ID != "20261027100000-test000" {
+ t.Fatalf("boxes=%v: count=%d, blocks=%+v", boxes, count, blocks)
+ }
+ }
+ for _, change := range []string{"model", "endpoint", "dimensions"} {
+ cfg := Conf.AI.Load().Embedding
+ cfg.Name, cfg.BaseURL, cfg.Dimensions = "model", server.URL, 2
+ switch change {
+ case "model":
+ cfg.Name = "new"
+ case "endpoint":
+ cfg.BaseURL += "/new"
+ case "dimensions":
+ cfg.Dimensions = 4
+ }
+ _, count, _, _ := SemanticSearchBlock("query", nil, nil, nil, nil, 1, 20, nil, nil)
+ if count != 0 {
+ t.Fatalf("%s mixed old vectors: %d", change, count)
+ }
+ }
+ Conf.AI.Load().Embedding.Name, Conf.AI.Load().Embedding.BaseURL, Conf.AI.Load().Embedding.Dimensions = "model", server.URL, 2
+ switchConfig.Store(true)
+ _, count, _, _ := SemanticSearchBlock("query", nil, nil, nil, nil, 1, 20, nil, nil)
+ if count != 0 || calls.Load() != 6 {
+ t.Fatalf("inflight config change: count=%d, calls=%d", count, calls.Load())
+ }
+ rows, err := sql.QueryNoLimit("SELECT COUNT(*) AS c FROM block_embeddings")
+ if err != nil || rows[0]["c"].(int64) != 6 {
+ t.Fatalf("old index changed: %v, %v", rows, err)
+ }
+}
diff --git a/kernel/model/embedding_store.go b/kernel/model/embedding_store.go
index f2d5fb0c38..c5f2a05fbe 100644
--- a/kernel/model/embedding_store.go
+++ b/kernel/model/embedding_store.go
@@ -2,8 +2,8 @@ package model
// 写入时在同一 SQL 中验证内容及更新时间,避免查询与写入之间发生编辑而产生过期索引。
const stmtStoreEmbedding = "INSERT OR REPLACE INTO block_embeddings " +
- "(id, root_id, box, path, embedding, model, content_len, updated, fail_count, last_tried, ignored_type) " +
- "SELECT id, root_id, box, path, ?, ?, ?, updated, 0, 0, 0 FROM blocks " +
+ "(id, root_id, box, path, embedding, model, space, content_len, updated, fail_count, last_tried, ignored_type) " +
+ "SELECT id, root_id, box, path, ?, ?, ?, ?, updated, 0, 0, 0 FROM blocks " +
"WHERE id = ? AND updated = ? AND content = ?"
// 失败仅累加空向量记录,成功结果不受迟到的失败影响。
diff --git a/kernel/model/embedding_store_test.go b/kernel/model/embedding_store_test.go
index 18e78589a2..51f15980ac 100644
--- a/kernel/model/embedding_store_test.go
+++ b/kernel/model/embedding_store_test.go
@@ -21,7 +21,7 @@ func TestEmbeddingResultWrites(t *testing.T) {
}
}
exec("CREATE TABLE blocks (id TEXT PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, updated TEXT, content TEXT)")
- exec("CREATE TABLE block_embeddings (id TEXT PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, embedding BLOB, model TEXT, content_len INTEGER, updated TEXT, fail_count INTEGER, last_tried INTEGER, ignored_type INTEGER)")
+ exec("CREATE TABLE block_embeddings (id TEXT PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, embedding BLOB, model TEXT, space TEXT NOT NULL DEFAULT '', content_len INTEGER, updated TEXT, fail_count INTEGER, last_tried INTEGER, ignored_type INTEGER)")
exec("INSERT INTO blocks VALUES ('id', 'root', 'box', '/path', '1', 'original')")
check := func(wantCount, wantFailures, wantBytes int) {
t.Helper()
@@ -42,14 +42,14 @@ func TestEmbeddingResultWrites(t *testing.T) {
if err := db.QueryRow("SELECT ignored_type FROM block_embeddings").Scan(&ignored); err != nil || ignored != 2 {
t.Fatalf("ignored=%d, error=%v", ignored, err)
}
- exec(stmtStoreEmbedding, []byte{1, 2, 3, 4}, "model", 8, "id", "1", "original")
+ exec(stmtStoreEmbedding, []byte{1, 2, 3, 4}, "model", "space", 8, "id", "1", "original")
exec(stmtFailedEmbedding, []byte{}, "model", 3, "id", "1", "original")
check(1, 0, 4)
// 同一秒内编辑也必须凭内容拦截旧结果。
exec("DELETE FROM block_embeddings")
exec("UPDATE blocks SET content = 'edited'")
- exec(stmtStoreEmbedding, []byte{1}, "model", 8, "id", "1", "original")
+ exec(stmtStoreEmbedding, []byte{1}, "model", "space", 8, "id", "1", "original")
exec(stmtFailedEmbedding, []byte{}, "model", 4, "id", "1", "original")
exec(stmtIgnoreEmbedding, []byte{}, "model", 1, "id", "1", "original", "box", "/path")
check(0, 0, 0)
@@ -58,14 +58,14 @@ func TestEmbeddingResultWrites(t *testing.T) {
exec("UPDATE blocks SET path = '/moved'")
exec(stmtIgnoreEmbedding, []byte{}, "model", 2, "id", "1", "edited", "box", "/path")
check(0, 0, 0)
- exec(stmtStoreEmbedding, []byte{1}, "model", 6, "id", "1", "edited")
+ exec(stmtStoreEmbedding, []byte{1}, "model", "space", 6, "id", "1", "edited")
var path string
if err := db.QueryRow("SELECT path FROM block_embeddings").Scan(&path); err != nil || path != "/moved" {
t.Fatalf("path=%s, error=%v", path, err)
}
exec("DELETE FROM blocks")
exec("DELETE FROM block_embeddings")
- exec(stmtStoreEmbedding, []byte{1}, "model", 6, "id", "1", "edited")
+ exec(stmtStoreEmbedding, []byte{1}, "model", "space", 6, "id", "1", "edited")
exec(stmtFailedEmbedding, []byte{}, "model", 5, "id", "1", "edited")
check(0, 0, 0)
}
diff --git a/kernel/sql/database.go b/kernel/sql/database.go
index 054b910a08..035cded689 100644
--- a/kernel/sql/database.go
+++ b/kernel/sql/database.go
@@ -311,7 +311,7 @@ func initDBTables() {
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "drop table [block_embeddings] failed: %s", err)
}
- _, err = db.Exec("CREATE TABLE block_embeddings (id TEXT PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, embedding BLOB, model TEXT, content_len INTEGER, updated TEXT, fail_count INTEGER NOT NULL DEFAULT 0, last_tried INTEGER NOT NULL DEFAULT 0, ignored_type INTEGER NOT NULL DEFAULT 0)")
+ _, err = db.Exec("CREATE TABLE block_embeddings (id TEXT PRIMARY KEY, root_id TEXT, box TEXT, path TEXT, embedding BLOB, model TEXT, space TEXT NOT NULL DEFAULT '', content_len INTEGER, updated TEXT, fail_count INTEGER NOT NULL DEFAULT 0, last_tried INTEGER NOT NULL DEFAULT 0, ignored_type INTEGER NOT NULL DEFAULT 0)")
if err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create table [block_embeddings] failed: %s", err)
}
@@ -1725,6 +1725,9 @@ func migrateBlockEmbeddingsSchema() {
addColumn("fail_count", addColumns[0])
addColumn("last_tried", addColumns[1])
addColumn("ignored_type", addColumns[2])
+ // Existing vectors have unknown provenance; preserve them without guessing
+ // a space or scheduling paid re-embedding. Users can rebuild explicitly.
+ addColumn("space", "ALTER TABLE block_embeddings ADD COLUMN space TEXT NOT NULL DEFAULT ''")
}
func beginTx() (tx *sql.Tx, err error) {
diff --git a/kernel/sql/embedding_schema_test.go b/kernel/sql/embedding_schema_test.go
new file mode 100644
index 0000000000..d4c0b11cac
--- /dev/null
+++ b/kernel/sql/embedding_schema_test.go
@@ -0,0 +1,35 @@
+package sql
+
+import (
+ stdsql "database/sql"
+ "testing"
+)
+
+func TestEmbeddingSpaceMigrationPreservesLegacy(t *testing.T) {
+ database, err := stdsql.Open("sqlite3_extended", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer database.Close()
+ database.SetMaxOpenConns(1)
+ previous := db
+ db = database
+ defer func() { db = previous }()
+ if _, err = database.Exec("CREATE TABLE block_embeddings (id TEXT PRIMARY KEY, embedding BLOB, model TEXT)"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = database.Exec("INSERT INTO block_embeddings VALUES ('legacy', x'01020304', 'old-model')"); err != nil {
+ t.Fatal(err)
+ }
+ for range 2 {
+ migrateBlockEmbeddingsSchema()
+ }
+ var vector []byte
+ var model, space string
+ if err = database.QueryRow("SELECT embedding, model, space FROM block_embeddings WHERE id = 'legacy'").Scan(&vector, &model, &space); err != nil {
+ t.Fatal(err)
+ }
+ if len(vector) != 4 || model != "old-model" || space != "" {
+ t.Fatalf("legacy result changed: %x, %q, %q", vector, model, space)
+ }
+}