Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(sqlite): 索引行写入失败不再被丢弃,BatchSave 回滚而不是留下「主行在、索引行不在」 (#3843)
linus-liu-weba8a2c17b892c
diff --git a/internal/application/repository/retriever/sqlite/index_write_errors_test.go b/internal/application/repository/retriever/sqlite/index_write_errors_test.gonew file mode 100644index 000000000..ba95980dd--- /dev/null+++ b/internal/application/repository/retriever/sqlite/index_write_errors_test.go@@ -0,0 +1,271 @@+package sqlite++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// A chunk row is only useful when its vec0 and FTS rows exist too: retrieval+// joins both index tables on lite_embeddings.id. A discarded index write used+// to commit the chunk row while Save still returned nil, so these tests pin the+// error and the rollback, not just the error.++func countSQLiteRows(t *testing.T, repository *sqliteRepository, sql string, args ...any) int64 {+ t.Helper()+ var count int64+ require.NoError(t, repository.db.Raw(sql, args...).Scan(&count).Error, "count query: %s", sql)+ return count+}++func TestBatchSaveRollsBackWhenVecTableIsGone(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ ctx := context.Background()++ control := sqliteTestIndex("ctl", "kb", "knowledge", "", true)+ saveSQLiteTestVector(t, repository, control, []float32{1, 0})+ require.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))++ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)++ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)+ err := repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},+ })++ require.Error(t, err, "a dropped vec0 table must not be reported as a successful save")+ assert.Contains(t, err.Error(), "vec_embeddings_2")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),+ "the chunk row must roll back together with its vector row")++ // The failed write drops the cached "ready" state, so the next save+ // recreates the table instead of failing forever.+ require.NoError(t, repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},+ }))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))+}++func TestBatchSaveRollsBackWhenFTSTableIsGone(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ if !repository.db.Migrator().HasTable("lite_embeddings_fts") {+ t.Skip("FTS5 unavailable (build without sqlite_fts5)")+ }+ ctx := context.Background()++ control := sqliteTestIndex("ctl", "kb", "knowledge", "", true)+ saveSQLiteTestVector(t, repository, control, []float32{1, 0})+ require.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM lite_embeddings_fts"))++ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings_fts").Error)++ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)+ err := repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},+ })++ require.Error(t, err, "a dropped FTS table must not be reported as a successful save")+ assert.Contains(t, err.Error(), "lite_embeddings_fts")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),+ "the chunk row must roll back together with its keyword index row")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository, `SELECT count(*) FROM vec_embeddings_2+ WHERE rowid IN (SELECT id FROM lite_embeddings WHERE chunk_id = ?)`, victim.ChunkID))+}++func TestBatchSaveRollsBackWhenVecInsertIsRejected(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ ctx := context.Background()++ // A vec0 table of another dimension occupies the name: the table is real,+ // so only the insert itself can fail. A 2-dimension vector into float[3]+ // is rejected by vec0.+ require.NoError(t, repository.db.Exec(`CREATE VIRTUAL TABLE vec_embeddings_2+ USING vec0(embedding float[3] distance_metric=cosine)`).Error)++ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)+ err := repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},+ })++ require.Error(t, err)+ assert.Contains(t, err.Error(), "vec_embeddings_2")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),+ "a rejected vector insert must roll the chunk row back")+}++// An object that only occupies the vec0 table name used to be cached as a+// ready dimension, which kept the whole process from ever creating the table.+func TestSaveFailsWhenAnotherObjectOwnsTheVecTableName(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ ctx := context.Background()++ require.NoError(t, repository.db.Exec(`CREATE VIEW vec_embeddings_2 AS SELECT 1 AS x WHERE 0`).Error)++ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)+ err := repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {1, 0}},+ })++ require.Error(t, err)+ assert.Contains(t, err.Error(), "vec_embeddings_2")+ assert.False(t, repository.hasVecTable(2), "a failed prepare must not be cached as ready")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID))++ // Once the obstruction is gone the very next save must retry the table.+ require.NoError(t, repository.db.Exec("DROP VIEW vec_embeddings_2").Error)+ require.NoError(t, repository.Save(ctx, victim, map[string]any{+ "embedding": map[string][]float32{victim.SourceID: {1, 0}},+ }))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM vec_embeddings_2"))+}++func TestDeleteByChunkIDListReportsIndexDeleteFailure(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})++ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)++ err := repository.DeleteByChunkIDList(context.Background(), []string{info.ChunkID}, 2,+ string(types.KnowledgeTypeManual))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "vec_embeddings_2")+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID),+ "the chunk row must survive an index delete that failed")+}++func TestCopyIndicesReportsVectorCopyFailure(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ source := sqliteTestIndex("chunk", "kb-src", "knowledge-src", "", true)+ saveSQLiteTestVector(t, repository, source, []float32{1, 0})++ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)++ err := repository.CopyIndices(context.Background(), "kb-src",+ map[string]string{"knowledge-src": "knowledge-dst"},+ map[string]string{"chunk": "chunk-dst"},+ "kb-dst", 2, string(types.KnowledgeTypeManual),+ )++ require.Error(t, err, "a copy without its vector row must not be reported as a success")+ assert.Contains(t, err.Error(), "vec_embeddings_2")+}++// A rejected row used to be logged and skipped, so a clone could report success+// while one chunk never became retrievable. The error must fail the copy and+// name the lost row, without quoting content.+func TestCopyIndicesReportsRejectedRowWithChunkIDs(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ chunk := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", true)+ chunk.SourceID = "src-chunk"+ chunk.Content = "secret chunk body"+ question := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", false)+ question.SourceID = "src-chunk-q1"+ question.Content = "secret question body"+ require.NoError(t, repository.BatchSave(context.Background(), []*types.IndexInfo{chunk, question}, map[string]any{+ "embedding": map[string][]float32{+ chunk.SourceID: {1, 0},+ question.SourceID: {0, 1},+ },+ }))++ // Only the chunk row is rejected; the question row of the same chunk still+ // has to be copied, so the error can be shown to point at one row.+ require.NoError(t, repository.db.Exec(`CREATE TRIGGER reject_copied_chunk BEFORE INSERT ON lite_embeddings+ WHEN NEW.chunk_id = 'dst-chunk' AND NEW.source_id = 'dst-chunk'+ BEGIN SELECT RAISE(ABORT, 'rejected copy'); END`).Error)++ err := repository.CopyIndices(context.Background(), "kb-src",+ map[string]string{"knowledge-src": "knowledge-dst"},+ map[string]string{"src-chunk": "dst-chunk"},+ "kb-dst", 2, string(types.KnowledgeTypeManual),+ )++ require.Error(t, err, "a rejected row must not be reported as a successful copy")+ assert.Contains(t, err.Error(), "src-chunk", "the error must name the source chunk")+ assert.Contains(t, err.Error(), "dst-chunk", "the error must name the target chunk")+ assert.NotContains(t, err.Error(), "secret", "the error must locate the row without leaking its content")+ assert.NotContains(t, err.Error(), "src-chunk-q1", "only the rejected row may be reported")+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk-q1"),+ "the sibling row of the same chunk must still be copied")+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk"))+}++// A failing keyword index write is accumulated instead of ending the copy: the+// caller sees the row that lost its index row, and the copy keeps going so one+// broken row cannot hide the remaining failures.+func TestCopyIndicesReportsKeywordCopyFailureWithoutLosingTheRow(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ if !repository.db.Migrator().HasTable("lite_embeddings_fts") {+ t.Skip("FTS5 unavailable (build without sqlite_fts5)")+ }++ source := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", true)+ source.SourceID = "src-chunk"+ saveSQLiteTestVector(t, repository, source, []float32{1, 0})+ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings_fts").Error)++ err := repository.CopyIndices(context.Background(), "kb-src",+ map[string]string{"knowledge-src": "knowledge-dst"},+ map[string]string{"src-chunk": "dst-chunk"},+ "kb-dst", 2, string(types.KnowledgeTypeManual),+ )++ require.Error(t, err, "a copy whose keyword index row was lost must not report success")+ assert.Contains(t, err.Error(), "src-chunk")+ assert.Contains(t, err.Error(), "dst-chunk")+ assert.Contains(t, err.Error(), "lite_embeddings_fts")+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk"))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ `SELECT count(*) FROM vec_embeddings_2 WHERE rowid IN+ (SELECT id FROM lite_embeddings WHERE source_id = ?)`, "dst-chunk"),+ "the vector row must still be copied after the keyword index write failed")+}++func TestSaveStillWritesEveryIndexRowOnHealthyTables(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})++ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))+ if repository.db.Migrator().HasTable("lite_embeddings_fts") {+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings_fts"))+ assert.Len(t, keywordSearch(t, repository, "content of chunk"), 1)+ }+}++// A driver built without the fts5 module cannot host a keyword index at all.+// That is a property of the build, so saving keeps working and the keyword+// index is simply absent; a table that goes missing while fts5 is compiled in+// is a real failure and is covered by TestBatchSaveRollsBackWhenFTSTableIsGone.+func TestSaveWorksWhenTheFTS5ModuleIsNotCompiledIn(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ if repository.db.Migrator().HasTable("lite_embeddings_fts") {+ t.Skip("fts5 is compiled in; the keyword index is checked by the other tests")+ }++ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})++ assert.EqualValues(t, 1, countSQLiteRows(t, repository,+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID))+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))+}diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.goindex 1973a8880..5622df031 100644--- a/internal/application/repository/retriever/sqlite/repository.go+++ b/internal/application/repository/retriever/sqlite/repository.go@@ -42,6 +42,11 @@ type sqliteRepository struct {// all consult it, and an unguarded map write is a fatal runtime error.vecMu sync.RWMutexvecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)+ // ftsUnavailable records that the driver was built without the fts5 module+ // (upstream test runs do not pass -tags sqlite_fts5). Keyword indexing+ // cannot exist in such a build at all, so its writes are skipped instead of+ // failing every ingestion; Lite release builds always pass the tag.+ ftsUnavailable bool}func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {@@ -51,14 +56,22 @@ func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineReplogger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)}- initFTS5(db)- enableImageChunkIndexes(db)-repo := &sqliteRepository{db: db,vecTables: make(map[int]bool),}+ if err := repo.initFTS5(); err != nil {+ if errors.Is(err, errFTS5ModuleMissing) {+ repo.ftsUnavailable = true+ logger.GetLogger(context.Background()).Warnf(+ "[SQLite] Keyword index disabled, driver has no fts5 module: %v", err)+ } else {+ logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to initialize FTS5 index: %v", err)+ }+ }+ enableImageChunkIndexes(db)+repo.ensureExistingVecTables()return repo@@ -86,28 +99,54 @@ func enableImageChunkIndexes(db *gorm.DB) {}}-func initFTS5(db *gorm.DB) {+// errFTS5ModuleMissing reports a driver built without the fts5 module. The+// keyword index cannot exist in such a build, which is a property of the build+// rather than a write that failed at runtime.+var errFTS5ModuleMissing = errors.New("sqlite driver built without the fts5 module")++// fts5InsertSQL is shared by the initial population and every later keyword+// index write, so both always build the same row shape.+const fts5InsertSQL = `INSERT INTO lite_embeddings_fts(+ rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`++// initFTS5 prepares the contentless FTS5 keyword index. A failure is returned+// so it is not mistaken for a working index; the table and its content are+// built in one transaction, because a half-populated index silently drops+// keyword recall for every row that was left out.+func (r *sqliteRepository) initFTS5() error {var sqlStr string- db.Raw("SELECT sql FROM sqlite_master WHERE type='table' AND name='lite_embeddings_fts'").Scan(&sqlStr)+ err := r.db.Raw(+ "SELECT sql FROM sqlite_master WHERE type='table' AND name='lite_embeddings_fts'",+ ).Scan(&sqlStr).Error+ if err != nil {+ return fmt.Errorf("[SQLite] failed to inspect lite_embeddings_fts: %w", err)+ }// If the table exists but is not contentless (i.e. uses content='lite_embeddings'), recreate it- if sqlStr != "" && strings.Contains(sqlStr, "content='lite_embeddings'") {- logger.GetLogger(context.Background()).Infof("[SQLite] Migrating FTS5 table to contentless table with manual bigram tokenization")- db.Exec("DROP TABLE IF EXISTS lite_embeddings_fts")- sqlStr = "" // Trigger recreate+ migrate := sqlStr != "" && strings.Contains(sqlStr, "content='lite_embeddings'")+ if migrate {+ logger.GetLogger(context.Background()).Infof(+ "[SQLite] Migrating FTS5 table to contentless table with manual bigram tokenization")+ } else if sqlStr != "" {+ return nil}- if sqlStr == "" {+ err = r.db.Transaction(func(tx *gorm.DB) error {+ if migrate {+ if err := tx.Exec("DROP TABLE IF EXISTS lite_embeddings_fts").Error; err != nil {+ return err+ }+ }createSQL := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(content, source_id, chunk_id, knowledge_id, knowledge_base_id,content='',contentless_delete=1,tokenize='unicode61')`- if err := db.Exec(createSQL).Error; err != nil {- logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)- return+ if err := tx.Exec(createSQL).Error; err != nil {+ return err}- logger.GetLogger(context.Background()).Infof("[SQLite] Populating contentless FTS5 table from lite_embeddings with bigrams")+ logger.GetLogger(context.Background()).Infof(+ "[SQLite] Populating contentless FTS5 table from lite_embeddings with bigrams")// To populate, we need to read all rows and insert them via Go to apply bigram tokenizationtype Row struct {@@ -119,13 +158,26 @@ func initFTS5(db *gorm.DB) {KnowledgeBaseID string}var rows []Row- db.Raw("SELECT id, content, source_id, chunk_id, knowledge_id, knowledge_base_id FROM lite_embeddings").Scan(&rows)- for _, r := range rows {- db.Exec(`INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id)- VALUES(?, ?, ?, ?, ?, ?)`,- r.ID, tokenizeCJKBigram(r.Content), r.SourceID, r.ChunkID, r.KnowledgeID, r.KnowledgeBaseID)+ if err := tx.Raw(+ "SELECT id, content, source_id, chunk_id, knowledge_id, knowledge_base_id FROM lite_embeddings",+ ).Scan(&rows).Error; err != nil {+ return err+ }+ for _, row := range rows {+ if err := tx.Exec(fts5InsertSQL, row.ID, tokenizeCJKBigram(row.Content), row.SourceID,+ row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID).Error; err != nil {+ return err+ }+ }+ return nil+ })+ if err != nil {+ if strings.Contains(err.Error(), "no such module: fts5") {+ return fmt.Errorf("%w: %v", errFTS5ModuleMissing, err)}+ return fmt.Errorf("[SQLite] failed to initialize lite_embeddings_fts: %w", err)}+ return nil}func vecTableName(dim int) string {@@ -144,9 +196,44 @@ func (r *sqliteRepository) markVecTable(dim int) {r.vecTables[dim] = true}-func (r *sqliteRepository) ensureVecTable(dim int) {- if dim <= 0 || r.hasVecTable(dim) {- return+// unmarkVecTable drops the cached "ready" state of a dimension after a write+// against its table failed, so the next write re-checks the table instead of+// trusting a marker that no longer describes it.+func (r *sqliteRepository) unmarkVecTable(dim int) {+ r.vecMu.Lock()+ defer r.vecMu.Unlock()+ delete(r.vecTables, dim)+}++// verifyVecTable rejects a same-named object that is not a vec0 table. It is+// needed because CREATE VIRTUAL TABLE IF NOT EXISTS is a silent no-op when any+// object already occupies the name, and a marker set from that no-op kept the+// dimension unindexed for the rest of the process lifetime.+func (r *sqliteRepository) verifyVecTable(tbl string) error {+ var createSQL string+ if err := r.db.Raw(+ `SELECT COALESCE(sql, '') FROM sqlite_master WHERE type = 'table' AND name = ?`, tbl,+ ).Scan(&createSQL).Error; err != nil {+ return fmt.Errorf("[SQLite] failed to inspect %s: %w", tbl, err)+ }+ if createSQL == "" {+ return fmt.Errorf("[SQLite] %s is missing after CREATE VIRTUAL TABLE", tbl)+ }+ if !strings.Contains(strings.ToLower(createSQL), "using vec0") {+ return fmt.Errorf("[SQLite] %s exists but is not a vec0 table: %s", tbl, createSQL)+ }+ return nil+}++// ensureVecTable makes the vec0 table of a dimension usable and returns an+// error when it cannot, so callers never write vectors into a table that does+// not exist. A dimension is only cached as ready once the table was verified.+func (r *sqliteRepository) ensureVecTable(dim int) error {+ if dim <= 0 {+ return nil+ }+ if r.hasVecTable(dim) {+ return nil}tbl := vecTableName(dim)createSQL := fmt.Sprintf(@@ -154,21 +241,29 @@ func (r *sqliteRepository) ensureVecTable(dim int) {tbl, dim,)if err := r.db.Exec(createSQL).Error; err != nil {- if strings.Contains(err.Error(), "already exists") {- r.markVecTable(dim)- return- }- logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)- return+ return fmt.Errorf("[SQLite] failed to create vec0 table %s: %w", tbl, err)+ }+ if err := r.verifyVecTable(tbl); err != nil {+ return err}r.markVecTable(dim)+ return nil}func (r *sqliteRepository) ensureExistingVecTables() {var dims []int- r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)+ if err := r.db.Model(&sqliteEmbedding{}).Distinct("dimension").+ Where("dimension > 0").Pluck("dimension", &dims).Error; err != nil {+ logger.GetLogger(context.Background()).Warnf(+ "[SQLite] Failed to list indexed dimensions: %v", err)+ return+ }for _, dim := range dims {- r.ensureVecTable(dim)+ // A failure here is not cached: the next write retries and reports it.+ if err := r.ensureVecTable(dim); err != nil {+ logger.GetLogger(context.Background()).Errorf(+ "[SQLite] Failed to prepare vec0 table for dim %d: %v", dim, err)+ }}}@@ -224,7 +319,9 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*typessourceIDs = append(sourceIDs, row.SourceID)// vec0 tables are created outside the transaction below: production// SQLite runs on one connection, which the transaction holds.- r.ensureVecTable(row.Dimension)+ if err := r.ensureVecTable(row.Dimension); err != nil {+ return err+ }}// Replacing a source is one transaction, so a failed insert cannot// leave the source with its old rows deleted and no new ones.@@ -242,7 +339,9 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types}}if len(existing) > 0 {- r.deleteRowsAndVecs(tx, existing)+ if err := r.deleteRowsAndVecs(tx, existing); err != nil {+ return err+ }if err := tx.Where("id IN ?", existingIDs).Delete(&sqliteEmbedding{}).Error; err != nil {return err}@@ -251,10 +350,18 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*typesif err := tx.Create(rows).Error; err != nil {return err}+ // Every index row is written in the same transaction as the chunk row:+ // returning the error rolls the chunk row back instead of committing a+ // row that neither retrieval path can find.for i, row := range rows {- r.syncFTS5Insert(tx, row)- if len(embs[i]) > 0 && row.ID > 0 {- r.insertVec(tx, row.ID, row.Dimension, embs[i])+ if err := r.syncFTS5Insert(tx, row); err != nil {+ return err+ }+ if len(embs[i]) > 0 {+ if err := r.insertVec(tx, row.ID, row.Dimension, embs[i]); err != nil {+ r.unmarkVecTable(row.Dimension)+ return err+ }}}return nil@@ -271,22 +378,34 @@ func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoListfunc (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {var rows []sqliteEmbedding- r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)+ if err := r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows).Error; err != nil {+ return err+ }+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {+ return err+ }return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error}func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {var rows []sqliteEmbedding- r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)+ if err := r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows).Error; err != nil {+ return err+ }+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {+ return err+ }return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error}func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {var rows []sqliteEmbedding- r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)+ if err := r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows).Error; err != nil {+ return err+ }+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {+ return err+ }return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error}@@ -302,6 +421,12 @@ func (r *sqliteRepository) CopyIndices(ctx context.Context,sourceChunkIDs = append(sourceChunkIDs, sourceChunkID)}const batchSize = 500+ // A copied chunk is only usable when its row, keyword index row and vector+ // row all land: retrieval joins both index tables on lite_embeddings.id, so+ // a dropped write leaves a chunk neither path can find while the copy still+ // reports success. Every failing row is collected and the loop keeps going,+ // so the caller learns all the rows that were lost, not just the first.+ var copyErrs []errorfor start := 0; start < len(sourceChunkIDs); start += batchSize {end := min(start+batchSize, len(sourceChunkIDs))// Every row of a chunk is copied: besides the chunk itself, generated@@ -326,17 +451,29 @@ func (r *sqliteRepository) CopyIndices(ctx context.Context,Dimension: src.Dimension,IsEnabled: src.IsEnabled,}+ // The source and target chunk IDs locate the lost row without+ // quoting its content into the error.if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {- logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy source %s: %v", src.SourceID, err)+ copyErrs = append(copyErrs, fmt.Errorf(+ "[SQLite] CopyIndices: failed to copy source %s (chunk %s -> %s): %w",+ src.SourceID, src.ChunkID, targetChunkID, err))continue}- r.syncFTS5Insert(r.db.WithContext(ctx), &newRow)- if src.Dimension > 0 && newRow.ID > 0 {- r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)+ if err := r.syncFTS5Insert(r.db.WithContext(ctx), &newRow); err != nil {+ copyErrs = append(copyErrs, fmt.Errorf(+ "[SQLite] CopyIndices: failed to copy keyword index of source %s (chunk %s -> %s): %w",+ src.SourceID, src.ChunkID, targetChunkID, err))+ }+ if src.Dimension > 0 {+ if err := r.copyVec(ctx, src.ID, newRow.ID, src.Dimension); err != nil {+ copyErrs = append(copyErrs, fmt.Errorf(+ "[SQLite] CopyIndices: failed to copy vector of source %s (chunk %s -> %s): %w",+ src.SourceID, src.ChunkID, targetChunkID, err))+ }}}}- return nil+ return errors.Join(copyErrs...)}// copiedSourceID maps a source row's SourceID onto the target chunk the way@@ -511,7 +648,9 @@ func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.Retr}dim := len(params.Embedding)- r.ensureVecTable(dim)+ if err := r.ensureVecTable(dim); err != nil {+ return nil, err+ }queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)if err != nil {@@ -633,17 +772,30 @@ func extractEmbedding(params map[string]any, sourceID string) []float32 {}// insertVec writes a row's vector through db. The vec0 table for dim must-// already exist (ensureVecTable), since db may be a transaction.-func (r *sqliteRepository) insertVec(db *gorm.DB, rowID uint, dim int, emb []float32) {+// already exist (ensureVecTable), since db may be a transaction. Failures are+// returned so the caller's transaction rolls back instead of committing a+// chunk row whose vector row was dropped.+func (r *sqliteRepository) insertVec(db *gorm.DB, rowID uint, dim int, emb []float32) error {+ if rowID == 0 {+ return errors.New("[SQLite] cannot insert a vector row without a row id")+ }blob, err := sqlite_vec.SerializeFloat32(emb)if err != nil {- return+ return fmt.Errorf("[SQLite] failed to serialize embedding of row %d: %w", rowID, err)+ }+ tbl := vecTableName(dim)+ if err := db.Exec(+ fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", tbl), rowID, blob,+ ).Error; err != nil {+ return fmt.Errorf("[SQLite] failed to insert vector row %d into %s: %w", rowID, tbl, err)}- sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))- db.Exec(sql, rowID, blob)+ return nil}-func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding) {+// deleteRowsAndVecs removes the vec0 and FTS copies of the given rows. Every+// failure is returned: deleting only the lite_embeddings rows would leave the+// retrieval indexes holding rows that no longer exist.+func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding) error {dimIDs := make(map[int][]uint)for _, row := range rows {if row.Dimension > 0 {@@ -656,32 +808,60 @@ func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding}tbl := vecTableName(dim)for _, id := range ids {- db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)+ if err := db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id).Error; err != nil {+ return fmt.Errorf("[SQLite] failed to delete vector row %d from %s: %w", id, tbl, err)+ }}}+ if r.ftsUnavailable {+ return nil+ }for _, row := range rows {- db.Exec("DELETE FROM lite_embeddings_fts WHERE rowid = ?", row.ID)+ if err := db.Exec("DELETE FROM lite_embeddings_fts WHERE rowid = ?", row.ID).Error; err != nil {+ return fmt.Errorf("[SQLite] failed to delete keyword index row %d: %w", row.ID, err)+ }}+ return nil}-func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {- if !r.hasVecTable(dim) {- return+// copyVec duplicates a vector row for a copied chunk. It reports an error when+// the destination table cannot be prepared or the source row has no vector+// row, so a clone cannot silently end up without vectors.+func (r *sqliteRepository) copyVec(ctx context.Context, srcID, dstID uint, dim int) error {+ if err := r.ensureVecTable(dim); err != nil {+ return err}tbl := vecTableName(dim)- r.db.Exec(fmt.Sprintf(+ result := r.db.WithContext(ctx).Exec(fmt.Sprintf("INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",tbl, tbl,), dstID, srcID)+ if result.Error != nil {+ r.unmarkVecTable(dim)+ return fmt.Errorf("[SQLite] failed to copy vector row %d -> %d in %s: %w",+ srcID, dstID, tbl, result.Error)+ }+ if result.RowsAffected == 0 {+ return fmt.Errorf("[SQLite] source row %d has no vector row in %s", srcID, tbl)+ }+ return nil}-func (r *sqliteRepository) syncFTS5Insert(db *gorm.DB, row *sqliteEmbedding) {+// syncFTS5Insert mirrors a lite_embeddings row into the keyword index. The+// error is returned so the caller's transaction rolls back instead of leaving+// the chunk unfindable by both retrieval paths.+func (r *sqliteRepository) syncFTS5Insert(db *gorm.DB, row *sqliteEmbedding) error {+ if r.ftsUnavailable {+ return nil+ }if row.ID == 0 {- return+ return fmt.Errorf("[SQLite] cannot index chunk %s: row has no id", row.SourceID)}- tokenizedContent := tokenizeCJKBigram(row.Content)- sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`- db.Exec(sql, row.ID, tokenizedContent, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)+ if err := db.Exec(fts5InsertSQL, row.ID, tokenizeCJKBigram(row.Content), row.SourceID, row.ChunkID,+ row.KnowledgeID, row.KnowledgeBaseID).Error; err != nil {+ return fmt.Errorf("[SQLite] failed to insert keyword index row for %s: %w", row.SourceID, err)+ }+ return nil}type whereClause struct {