Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(sqlite): 索引行写入失败不再被丢弃,BatchSave 回滚而不是留下「主行在、索引行不在」 (#3843)

linus-liu-weba8a2c17b892c
diff --git a/internal/application/repository/retriever/sqlite/index_write_errors_test.go b/internal/application/repository/retriever/sqlite/index_write_errors_test.go
new file mode 100644
index 000000000..ba95980dd
--- /dev/null
+++ b/internal/application/repository/retriever/sqlite/index_write_errors_test.go
@@ -0,0 +1,271 @@
+package sqlite
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// A chunk row is only useful when its vec0 and FTS rows exist too: retrieval
+// joins both index tables on lite_embeddings.id. A discarded index write used
+// to commit the chunk row while Save still returned nil, so these tests pin the
+// error and the rollback, not just the error.
+
+func countSQLiteRows(t *testing.T, repository *sqliteRepository, sql string, args ...any) int64 {
+ t.Helper()
+ var count int64
+ require.NoError(t, repository.db.Raw(sql, args...).Scan(&count).Error, "count query: %s", sql)
+ return count
+}
+
+func TestBatchSaveRollsBackWhenVecTableIsGone(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ ctx := context.Background()
+
+ control := sqliteTestIndex("ctl", "kb", "knowledge", "", true)
+ saveSQLiteTestVector(t, repository, control, []float32{1, 0})
+ require.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))
+
+ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)
+
+ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)
+ err := repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},
+ })
+
+ require.Error(t, err, "a dropped vec0 table must not be reported as a successful save")
+ assert.Contains(t, err.Error(), "vec_embeddings_2")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),
+ "the chunk row must roll back together with its vector row")
+
+ // The failed write drops the cached "ready" state, so the next save
+ // recreates the table instead of failing forever.
+ require.NoError(t, repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},
+ }))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))
+}
+
+func TestBatchSaveRollsBackWhenFTSTableIsGone(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ if !repository.db.Migrator().HasTable("lite_embeddings_fts") {
+ t.Skip("FTS5 unavailable (build without sqlite_fts5)")
+ }
+ ctx := context.Background()
+
+ control := sqliteTestIndex("ctl", "kb", "knowledge", "", true)
+ saveSQLiteTestVector(t, repository, control, []float32{1, 0})
+ require.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM lite_embeddings_fts"))
+
+ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings_fts").Error)
+
+ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)
+ err := repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},
+ })
+
+ require.Error(t, err, "a dropped FTS table must not be reported as a successful save")
+ assert.Contains(t, err.Error(), "lite_embeddings_fts")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),
+ "the chunk row must roll back together with its keyword index row")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository, `SELECT count(*) FROM vec_embeddings_2
+ WHERE rowid IN (SELECT id FROM lite_embeddings WHERE chunk_id = ?)`, victim.ChunkID))
+}
+
+func TestBatchSaveRollsBackWhenVecInsertIsRejected(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ ctx := context.Background()
+
+ // A vec0 table of another dimension occupies the name: the table is real,
+ // so only the insert itself can fail. A 2-dimension vector into float[3]
+ // is rejected by vec0.
+ require.NoError(t, repository.db.Exec(`CREATE VIRTUAL TABLE vec_embeddings_2
+ USING vec0(embedding float[3] distance_metric=cosine)`).Error)
+
+ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)
+ err := repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {0, 1}},
+ })
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "vec_embeddings_2")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID),
+ "a rejected vector insert must roll the chunk row back")
+}
+
+// An object that only occupies the vec0 table name used to be cached as a
+// ready dimension, which kept the whole process from ever creating the table.
+func TestSaveFailsWhenAnotherObjectOwnsTheVecTableName(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ ctx := context.Background()
+
+ require.NoError(t, repository.db.Exec(`CREATE VIEW vec_embeddings_2 AS SELECT 1 AS x WHERE 0`).Error)
+
+ victim := sqliteTestIndex("victim", "kb", "knowledge", "", true)
+ err := repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {1, 0}},
+ })
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "vec_embeddings_2")
+ assert.False(t, repository.hasVecTable(2), "a failed prepare must not be cached as ready")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", victim.ChunkID))
+
+ // Once the obstruction is gone the very next save must retry the table.
+ require.NoError(t, repository.db.Exec("DROP VIEW vec_embeddings_2").Error)
+ require.NoError(t, repository.Save(ctx, victim, map[string]any{
+ "embedding": map[string][]float32{victim.SourceID: {1, 0}},
+ }))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM vec_embeddings_2"))
+}
+
+func TestDeleteByChunkIDListReportsIndexDeleteFailure(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)
+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})
+
+ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)
+
+ err := repository.DeleteByChunkIDList(context.Background(), []string{info.ChunkID}, 2,
+ string(types.KnowledgeTypeManual))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "vec_embeddings_2")
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID),
+ "the chunk row must survive an index delete that failed")
+}
+
+func TestCopyIndicesReportsVectorCopyFailure(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ source := sqliteTestIndex("chunk", "kb-src", "knowledge-src", "", true)
+ saveSQLiteTestVector(t, repository, source, []float32{1, 0})
+
+ require.NoError(t, repository.db.Exec("DROP TABLE vec_embeddings_2").Error)
+
+ err := repository.CopyIndices(context.Background(), "kb-src",
+ map[string]string{"knowledge-src": "knowledge-dst"},
+ map[string]string{"chunk": "chunk-dst"},
+ "kb-dst", 2, string(types.KnowledgeTypeManual),
+ )
+
+ require.Error(t, err, "a copy without its vector row must not be reported as a success")
+ assert.Contains(t, err.Error(), "vec_embeddings_2")
+}
+
+// A rejected row used to be logged and skipped, so a clone could report success
+// while one chunk never became retrievable. The error must fail the copy and
+// name the lost row, without quoting content.
+func TestCopyIndicesReportsRejectedRowWithChunkIDs(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ chunk := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", true)
+ chunk.SourceID = "src-chunk"
+ chunk.Content = "secret chunk body"
+ question := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", false)
+ question.SourceID = "src-chunk-q1"
+ question.Content = "secret question body"
+ require.NoError(t, repository.BatchSave(context.Background(), []*types.IndexInfo{chunk, question}, map[string]any{
+ "embedding": map[string][]float32{
+ chunk.SourceID: {1, 0},
+ question.SourceID: {0, 1},
+ },
+ }))
+
+ // Only the chunk row is rejected; the question row of the same chunk still
+ // has to be copied, so the error can be shown to point at one row.
+ require.NoError(t, repository.db.Exec(`CREATE TRIGGER reject_copied_chunk BEFORE INSERT ON lite_embeddings
+ WHEN NEW.chunk_id = 'dst-chunk' AND NEW.source_id = 'dst-chunk'
+ BEGIN SELECT RAISE(ABORT, 'rejected copy'); END`).Error)
+
+ err := repository.CopyIndices(context.Background(), "kb-src",
+ map[string]string{"knowledge-src": "knowledge-dst"},
+ map[string]string{"src-chunk": "dst-chunk"},
+ "kb-dst", 2, string(types.KnowledgeTypeManual),
+ )
+
+ require.Error(t, err, "a rejected row must not be reported as a successful copy")
+ assert.Contains(t, err.Error(), "src-chunk", "the error must name the source chunk")
+ assert.Contains(t, err.Error(), "dst-chunk", "the error must name the target chunk")
+ assert.NotContains(t, err.Error(), "secret", "the error must locate the row without leaking its content")
+ assert.NotContains(t, err.Error(), "src-chunk-q1", "only the rejected row may be reported")
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk-q1"),
+ "the sibling row of the same chunk must still be copied")
+ assert.EqualValues(t, 0, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk"))
+}
+
+// A failing keyword index write is accumulated instead of ending the copy: the
+// caller sees the row that lost its index row, and the copy keeps going so one
+// broken row cannot hide the remaining failures.
+func TestCopyIndicesReportsKeywordCopyFailureWithoutLosingTheRow(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ if !repository.db.Migrator().HasTable("lite_embeddings_fts") {
+ t.Skip("FTS5 unavailable (build without sqlite_fts5)")
+ }
+
+ source := sqliteTestIndex("src-chunk", "kb-src", "knowledge-src", "", true)
+ source.SourceID = "src-chunk"
+ saveSQLiteTestVector(t, repository, source, []float32{1, 0})
+ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings_fts").Error)
+
+ err := repository.CopyIndices(context.Background(), "kb-src",
+ map[string]string{"knowledge-src": "knowledge-dst"},
+ map[string]string{"src-chunk": "dst-chunk"},
+ "kb-dst", 2, string(types.KnowledgeTypeManual),
+ )
+
+ require.Error(t, err, "a copy whose keyword index row was lost must not report success")
+ assert.Contains(t, err.Error(), "src-chunk")
+ assert.Contains(t, err.Error(), "dst-chunk")
+ assert.Contains(t, err.Error(), "lite_embeddings_fts")
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE source_id = ?", "dst-chunk"))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ `SELECT count(*) FROM vec_embeddings_2 WHERE rowid IN
+ (SELECT id FROM lite_embeddings WHERE source_id = ?)`, "dst-chunk"),
+ "the vector row must still be copied after the keyword index write failed")
+}
+
+func TestSaveStillWritesEveryIndexRowOnHealthyTables(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)
+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})
+
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))
+ if repository.db.Migrator().HasTable("lite_embeddings_fts") {
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings_fts"))
+ assert.Len(t, keywordSearch(t, repository, "content of chunk"), 1)
+ }
+}
+
+// A driver built without the fts5 module cannot host a keyword index at all.
+// That is a property of the build, so saving keeps working and the keyword
+// index is simply absent; a table that goes missing while fts5 is compiled in
+// is a real failure and is covered by TestBatchSaveRollsBackWhenFTSTableIsGone.
+func TestSaveWorksWhenTheFTS5ModuleIsNotCompiledIn(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ if repository.db.Migrator().HasTable("lite_embeddings_fts") {
+ t.Skip("fts5 is compiled in; the keyword index is checked by the other tests")
+ }
+
+ info := sqliteTestIndex("chunk", "kb", "knowledge", "", true)
+ saveSQLiteTestVector(t, repository, info, []float32{1, 0})
+
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository,
+ "SELECT count(*) FROM lite_embeddings WHERE chunk_id = ?", info.ChunkID))
+ assert.EqualValues(t, 1, countSQLiteRows(t, repository, "SELECT count(*) FROM vec_embeddings_2"))
+}
diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.go
index 1973a8880..5622df031 100644
--- a/internal/application/repository/retriever/sqlite/repository.go
+++ b/internal/application/repository/retriever/sqlite/repository.go
@@ -42,6 +42,11 @@ type sqliteRepository struct {
// all consult it, and an unguarded map write is a fatal runtime error.
vecMu sync.RWMutex
vecTables map[int]bool // tracks which vec0 tables have been created (keyed by dimension)
+ // ftsUnavailable records that the driver was built without the fts5 module
+ // (upstream test runs do not pass -tags sqlite_fts5). Keyword indexing
+ // cannot exist in such a build at all, so its writes are skipped instead of
+ // failing every ingestion; Lite release builds always pass the tag.
+ ftsUnavailable bool
}
func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository {
@@ -51,14 +56,22 @@ func NewSQLiteRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRep
logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to auto-migrate lite_embeddings: %v", err)
}
- initFTS5(db)
- enableImageChunkIndexes(db)
-
repo := &sqliteRepository{
db: db,
vecTables: make(map[int]bool),
}
+ if err := repo.initFTS5(); err != nil {
+ if errors.Is(err, errFTS5ModuleMissing) {
+ repo.ftsUnavailable = true
+ logger.GetLogger(context.Background()).Warnf(
+ "[SQLite] Keyword index disabled, driver has no fts5 module: %v", err)
+ } else {
+ logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to initialize FTS5 index: %v", err)
+ }
+ }
+ enableImageChunkIndexes(db)
+
repo.ensureExistingVecTables()
return repo
@@ -86,28 +99,54 @@ func enableImageChunkIndexes(db *gorm.DB) {
}
}
-func initFTS5(db *gorm.DB) {
+// errFTS5ModuleMissing reports a driver built without the fts5 module. The
+// keyword index cannot exist in such a build, which is a property of the build
+// rather than a write that failed at runtime.
+var errFTS5ModuleMissing = errors.New("sqlite driver built without the fts5 module")
+
+// fts5InsertSQL is shared by the initial population and every later keyword
+// index write, so both always build the same row shape.
+const fts5InsertSQL = `INSERT INTO lite_embeddings_fts(
+ rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
+
+// initFTS5 prepares the contentless FTS5 keyword index. A failure is returned
+// so it is not mistaken for a working index; the table and its content are
+// built in one transaction, because a half-populated index silently drops
+// keyword recall for every row that was left out.
+func (r *sqliteRepository) initFTS5() error {
var sqlStr string
- db.Raw("SELECT sql FROM sqlite_master WHERE type='table' AND name='lite_embeddings_fts'").Scan(&sqlStr)
+ err := r.db.Raw(
+ "SELECT sql FROM sqlite_master WHERE type='table' AND name='lite_embeddings_fts'",
+ ).Scan(&sqlStr).Error
+ if err != nil {
+ return fmt.Errorf("[SQLite] failed to inspect lite_embeddings_fts: %w", err)
+ }
// If the table exists but is not contentless (i.e. uses content='lite_embeddings'), recreate it
- if sqlStr != "" && strings.Contains(sqlStr, "content='lite_embeddings'") {
- logger.GetLogger(context.Background()).Infof("[SQLite] Migrating FTS5 table to contentless table with manual bigram tokenization")
- db.Exec("DROP TABLE IF EXISTS lite_embeddings_fts")
- sqlStr = "" // Trigger recreate
+ migrate := sqlStr != "" && strings.Contains(sqlStr, "content='lite_embeddings'")
+ if migrate {
+ logger.GetLogger(context.Background()).Infof(
+ "[SQLite] Migrating FTS5 table to contentless table with manual bigram tokenization")
+ } else if sqlStr != "" {
+ return nil
}
- if sqlStr == "" {
+ err = r.db.Transaction(func(tx *gorm.DB) error {
+ if migrate {
+ if err := tx.Exec("DROP TABLE IF EXISTS lite_embeddings_fts").Error; err != nil {
+ return err
+ }
+ }
createSQL := `CREATE VIRTUAL TABLE IF NOT EXISTS lite_embeddings_fts USING fts5(
content, source_id, chunk_id, knowledge_id, knowledge_base_id,
content='',
contentless_delete=1,
tokenize='unicode61'
)`
- if err := db.Exec(createSQL).Error; err != nil {
- logger.GetLogger(context.Background()).Warnf("[SQLite] Failed to create FTS5 table: %v", err)
- return
+ if err := tx.Exec(createSQL).Error; err != nil {
+ return err
}
- logger.GetLogger(context.Background()).Infof("[SQLite] Populating contentless FTS5 table from lite_embeddings with bigrams")
+ logger.GetLogger(context.Background()).Infof(
+ "[SQLite] Populating contentless FTS5 table from lite_embeddings with bigrams")
// To populate, we need to read all rows and insert them via Go to apply bigram tokenization
type Row struct {
@@ -119,13 +158,26 @@ func initFTS5(db *gorm.DB) {
KnowledgeBaseID string
}
var rows []Row
- db.Raw("SELECT id, content, source_id, chunk_id, knowledge_id, knowledge_base_id FROM lite_embeddings").Scan(&rows)
- for _, r := range rows {
- db.Exec(`INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id)
- VALUES(?, ?, ?, ?, ?, ?)`,
- r.ID, tokenizeCJKBigram(r.Content), r.SourceID, r.ChunkID, r.KnowledgeID, r.KnowledgeBaseID)
+ if err := tx.Raw(
+ "SELECT id, content, source_id, chunk_id, knowledge_id, knowledge_base_id FROM lite_embeddings",
+ ).Scan(&rows).Error; err != nil {
+ return err
+ }
+ for _, row := range rows {
+ if err := tx.Exec(fts5InsertSQL, row.ID, tokenizeCJKBigram(row.Content), row.SourceID,
+ row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ if strings.Contains(err.Error(), "no such module: fts5") {
+ return fmt.Errorf("%w: %v", errFTS5ModuleMissing, err)
}
+ return fmt.Errorf("[SQLite] failed to initialize lite_embeddings_fts: %w", err)
}
+ return nil
}
func vecTableName(dim int) string {
@@ -144,9 +196,44 @@ func (r *sqliteRepository) markVecTable(dim int) {
r.vecTables[dim] = true
}
-func (r *sqliteRepository) ensureVecTable(dim int) {
- if dim <= 0 || r.hasVecTable(dim) {
- return
+// unmarkVecTable drops the cached "ready" state of a dimension after a write
+// against its table failed, so the next write re-checks the table instead of
+// trusting a marker that no longer describes it.
+func (r *sqliteRepository) unmarkVecTable(dim int) {
+ r.vecMu.Lock()
+ defer r.vecMu.Unlock()
+ delete(r.vecTables, dim)
+}
+
+// verifyVecTable rejects a same-named object that is not a vec0 table. It is
+// needed because CREATE VIRTUAL TABLE IF NOT EXISTS is a silent no-op when any
+// object already occupies the name, and a marker set from that no-op kept the
+// dimension unindexed for the rest of the process lifetime.
+func (r *sqliteRepository) verifyVecTable(tbl string) error {
+ var createSQL string
+ if err := r.db.Raw(
+ `SELECT COALESCE(sql, '') FROM sqlite_master WHERE type = 'table' AND name = ?`, tbl,
+ ).Scan(&createSQL).Error; err != nil {
+ return fmt.Errorf("[SQLite] failed to inspect %s: %w", tbl, err)
+ }
+ if createSQL == "" {
+ return fmt.Errorf("[SQLite] %s is missing after CREATE VIRTUAL TABLE", tbl)
+ }
+ if !strings.Contains(strings.ToLower(createSQL), "using vec0") {
+ return fmt.Errorf("[SQLite] %s exists but is not a vec0 table: %s", tbl, createSQL)
+ }
+ return nil
+}
+
+// ensureVecTable makes the vec0 table of a dimension usable and returns an
+// error when it cannot, so callers never write vectors into a table that does
+// not exist. A dimension is only cached as ready once the table was verified.
+func (r *sqliteRepository) ensureVecTable(dim int) error {
+ if dim <= 0 {
+ return nil
+ }
+ if r.hasVecTable(dim) {
+ return nil
}
tbl := vecTableName(dim)
createSQL := fmt.Sprintf(
@@ -154,21 +241,29 @@ func (r *sqliteRepository) ensureVecTable(dim int) {
tbl, dim,
)
if err := r.db.Exec(createSQL).Error; err != nil {
- if strings.Contains(err.Error(), "already exists") {
- r.markVecTable(dim)
- return
- }
- logger.GetLogger(context.Background()).Errorf("[SQLite] Failed to create vec0 table for dim %d: %v", dim, err)
- return
+ return fmt.Errorf("[SQLite] failed to create vec0 table %s: %w", tbl, err)
+ }
+ if err := r.verifyVecTable(tbl); err != nil {
+ return err
}
r.markVecTable(dim)
+ return nil
}
func (r *sqliteRepository) ensureExistingVecTables() {
var dims []int
- r.db.Model(&sqliteEmbedding{}).Distinct("dimension").Where("dimension > 0").Pluck("dimension", &dims)
+ if err := r.db.Model(&sqliteEmbedding{}).Distinct("dimension").
+ Where("dimension > 0").Pluck("dimension", &dims).Error; err != nil {
+ logger.GetLogger(context.Background()).Warnf(
+ "[SQLite] Failed to list indexed dimensions: %v", err)
+ return
+ }
for _, dim := range dims {
- r.ensureVecTable(dim)
+ // A failure here is not cached: the next write retries and reports it.
+ if err := r.ensureVecTable(dim); err != nil {
+ logger.GetLogger(context.Background()).Errorf(
+ "[SQLite] Failed to prepare vec0 table for dim %d: %v", dim, err)
+ }
}
}
@@ -224,7 +319,9 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types
sourceIDs = append(sourceIDs, row.SourceID)
// vec0 tables are created outside the transaction below: production
// SQLite runs on one connection, which the transaction holds.
- r.ensureVecTable(row.Dimension)
+ if err := r.ensureVecTable(row.Dimension); err != nil {
+ return err
+ }
}
// Replacing a source is one transaction, so a failed insert cannot
// leave the source with its old rows deleted and no new ones.
@@ -242,7 +339,9 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types
}
}
if len(existing) > 0 {
- r.deleteRowsAndVecs(tx, existing)
+ if err := r.deleteRowsAndVecs(tx, existing); err != nil {
+ return err
+ }
if err := tx.Where("id IN ?", existingIDs).Delete(&sqliteEmbedding{}).Error; err != nil {
return err
}
@@ -251,10 +350,18 @@ func (r *sqliteRepository) BatchSave(ctx context.Context, indexInfoList []*types
if err := tx.Create(rows).Error; err != nil {
return err
}
+ // Every index row is written in the same transaction as the chunk row:
+ // returning the error rolls the chunk row back instead of committing a
+ // row that neither retrieval path can find.
for i, row := range rows {
- r.syncFTS5Insert(tx, row)
- if len(embs[i]) > 0 && row.ID > 0 {
- r.insertVec(tx, row.ID, row.Dimension, embs[i])
+ if err := r.syncFTS5Insert(tx, row); err != nil {
+ return err
+ }
+ if len(embs[i]) > 0 {
+ if err := r.insertVec(tx, row.ID, row.Dimension, embs[i]); err != nil {
+ r.unmarkVecTable(row.Dimension)
+ return err
+ }
}
}
return nil
@@ -271,22 +378,34 @@ func (r *sqliteRepository) EstimateStorageSize(_ context.Context, indexInfoList
func (r *sqliteRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
- r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows)
- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)
+ if err := r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Find(&rows).Error; err != nil {
+ return err
+ }
+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {
+ return err
+ }
return r.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
- r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows)
- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)
+ if err := r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Find(&rows).Error; err != nil {
+ return err
+ }
+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {
+ return err
+ }
return r.db.WithContext(ctx).Where("source_id IN ?", sourceIDList).Delete(&sqliteEmbedding{}).Error
}
func (r *sqliteRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, _ int, _ string) error {
var rows []sqliteEmbedding
- r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows)
- r.deleteRowsAndVecs(r.db.WithContext(ctx), rows)
+ if err := r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Find(&rows).Error; err != nil {
+ return err
+ }
+ if err := r.deleteRowsAndVecs(r.db.WithContext(ctx), rows); err != nil {
+ return err
+ }
return r.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&sqliteEmbedding{}).Error
}
@@ -302,6 +421,12 @@ func (r *sqliteRepository) CopyIndices(ctx context.Context,
sourceChunkIDs = append(sourceChunkIDs, sourceChunkID)
}
const batchSize = 500
+ // A copied chunk is only usable when its row, keyword index row and vector
+ // row all land: retrieval joins both index tables on lite_embeddings.id, so
+ // a dropped write leaves a chunk neither path can find while the copy still
+ // reports success. Every failing row is collected and the loop keeps going,
+ // so the caller learns all the rows that were lost, not just the first.
+ var copyErrs []error
for start := 0; start < len(sourceChunkIDs); start += batchSize {
end := min(start+batchSize, len(sourceChunkIDs))
// Every row of a chunk is copied: besides the chunk itself, generated
@@ -326,17 +451,29 @@ func (r *sqliteRepository) CopyIndices(ctx context.Context,
Dimension: src.Dimension,
IsEnabled: src.IsEnabled,
}
+ // The source and target chunk IDs locate the lost row without
+ // quoting its content into the error.
if err := r.db.WithContext(ctx).Create(&newRow).Error; err != nil {
- logger.GetLogger(ctx).Warnf("[SQLite] CopyIndices: failed to copy source %s: %v", src.SourceID, err)
+ copyErrs = append(copyErrs, fmt.Errorf(
+ "[SQLite] CopyIndices: failed to copy source %s (chunk %s -> %s): %w",
+ src.SourceID, src.ChunkID, targetChunkID, err))
continue
}
- r.syncFTS5Insert(r.db.WithContext(ctx), &newRow)
- if src.Dimension > 0 && newRow.ID > 0 {
- r.copyVec(ctx, src.ID, newRow.ID, src.Dimension)
+ if err := r.syncFTS5Insert(r.db.WithContext(ctx), &newRow); err != nil {
+ copyErrs = append(copyErrs, fmt.Errorf(
+ "[SQLite] CopyIndices: failed to copy keyword index of source %s (chunk %s -> %s): %w",
+ src.SourceID, src.ChunkID, targetChunkID, err))
+ }
+ if src.Dimension > 0 {
+ if err := r.copyVec(ctx, src.ID, newRow.ID, src.Dimension); err != nil {
+ copyErrs = append(copyErrs, fmt.Errorf(
+ "[SQLite] CopyIndices: failed to copy vector of source %s (chunk %s -> %s): %w",
+ src.SourceID, src.ChunkID, targetChunkID, err))
+ }
}
}
}
- return nil
+ return errors.Join(copyErrs...)
}
// copiedSourceID maps a source row's SourceID onto the target chunk the way
@@ -511,7 +648,9 @@ func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.Retr
}
dim := len(params.Embedding)
- r.ensureVecTable(dim)
+ if err := r.ensureVecTable(dim); err != nil {
+ return nil, err
+ }
queryBlob, err := sqlite_vec.SerializeFloat32(params.Embedding)
if err != nil {
@@ -633,17 +772,30 @@ func extractEmbedding(params map[string]any, sourceID string) []float32 {
}
// insertVec writes a row's vector through db. The vec0 table for dim must
-// already exist (ensureVecTable), since db may be a transaction.
-func (r *sqliteRepository) insertVec(db *gorm.DB, rowID uint, dim int, emb []float32) {
+// already exist (ensureVecTable), since db may be a transaction. Failures are
+// returned so the caller's transaction rolls back instead of committing a
+// chunk row whose vector row was dropped.
+func (r *sqliteRepository) insertVec(db *gorm.DB, rowID uint, dim int, emb []float32) error {
+ if rowID == 0 {
+ return errors.New("[SQLite] cannot insert a vector row without a row id")
+ }
blob, err := sqlite_vec.SerializeFloat32(emb)
if err != nil {
- return
+ return fmt.Errorf("[SQLite] failed to serialize embedding of row %d: %w", rowID, err)
+ }
+ tbl := vecTableName(dim)
+ if err := db.Exec(
+ fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", tbl), rowID, blob,
+ ).Error; err != nil {
+ return fmt.Errorf("[SQLite] failed to insert vector row %d into %s: %w", rowID, tbl, err)
}
- sql := fmt.Sprintf("INSERT INTO %s(rowid, embedding) VALUES (?, ?)", vecTableName(dim))
- db.Exec(sql, rowID, blob)
+ return nil
}
-func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding) {
+// deleteRowsAndVecs removes the vec0 and FTS copies of the given rows. Every
+// failure is returned: deleting only the lite_embeddings rows would leave the
+// retrieval indexes holding rows that no longer exist.
+func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding) error {
dimIDs := make(map[int][]uint)
for _, row := range rows {
if row.Dimension > 0 {
@@ -656,32 +808,60 @@ func (r *sqliteRepository) deleteRowsAndVecs(db *gorm.DB, rows []sqliteEmbedding
}
tbl := vecTableName(dim)
for _, id := range ids {
- db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id)
+ if err := db.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = ?", tbl), id).Error; err != nil {
+ return fmt.Errorf("[SQLite] failed to delete vector row %d from %s: %w", id, tbl, err)
+ }
}
}
+ if r.ftsUnavailable {
+ return nil
+ }
for _, row := range rows {
- db.Exec("DELETE FROM lite_embeddings_fts WHERE rowid = ?", row.ID)
+ if err := db.Exec("DELETE FROM lite_embeddings_fts WHERE rowid = ?", row.ID).Error; err != nil {
+ return fmt.Errorf("[SQLite] failed to delete keyword index row %d: %w", row.ID, err)
+ }
}
+ return nil
}
-func (r *sqliteRepository) copyVec(_ context.Context, srcID, dstID uint, dim int) {
- if !r.hasVecTable(dim) {
- return
+// copyVec duplicates a vector row for a copied chunk. It reports an error when
+// the destination table cannot be prepared or the source row has no vector
+// row, so a clone cannot silently end up without vectors.
+func (r *sqliteRepository) copyVec(ctx context.Context, srcID, dstID uint, dim int) error {
+ if err := r.ensureVecTable(dim); err != nil {
+ return err
}
tbl := vecTableName(dim)
- r.db.Exec(fmt.Sprintf(
+ result := r.db.WithContext(ctx).Exec(fmt.Sprintf(
"INSERT INTO %s(rowid, embedding) SELECT ?, embedding FROM %s WHERE rowid = ?",
tbl, tbl,
), dstID, srcID)
+ if result.Error != nil {
+ r.unmarkVecTable(dim)
+ return fmt.Errorf("[SQLite] failed to copy vector row %d -> %d in %s: %w",
+ srcID, dstID, tbl, result.Error)
+ }
+ if result.RowsAffected == 0 {
+ return fmt.Errorf("[SQLite] source row %d has no vector row in %s", srcID, tbl)
+ }
+ return nil
}
-func (r *sqliteRepository) syncFTS5Insert(db *gorm.DB, row *sqliteEmbedding) {
+// syncFTS5Insert mirrors a lite_embeddings row into the keyword index. The
+// error is returned so the caller's transaction rolls back instead of leaving
+// the chunk unfindable by both retrieval paths.
+func (r *sqliteRepository) syncFTS5Insert(db *gorm.DB, row *sqliteEmbedding) error {
+ if r.ftsUnavailable {
+ return nil
+ }
if row.ID == 0 {
- return
+ return fmt.Errorf("[SQLite] cannot index chunk %s: row has no id", row.SourceID)
}
- tokenizedContent := tokenizeCJKBigram(row.Content)
- sql := `INSERT INTO lite_embeddings_fts(rowid, content, source_id, chunk_id, knowledge_id, knowledge_base_id) VALUES(?, ?, ?, ?, ?, ?)`
- db.Exec(sql, row.ID, tokenizedContent, row.SourceID, row.ChunkID, row.KnowledgeID, row.KnowledgeBaseID)
+ if err := db.Exec(fts5InsertSQL, row.ID, tokenizeCJKBigram(row.Content), row.SourceID, row.ChunkID,
+ row.KnowledgeID, row.KnowledgeBaseID).Error; err != nil {
+ return fmt.Errorf("[SQLite] failed to insert keyword index row for %s: %w", row.SourceID, err)
+ }
+ return nil
}
type whereClause struct {