Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(sqlite): 检索后端批量更新索引副本失败不再吞掉错误 (#3768)
linus-liu-web72b8a106c77a
diff --git a/internal/application/repository/retriever/sqlite/batch_update_test.go b/internal/application/repository/retriever/sqlite/batch_update_test.gonew file mode 100644index 000000000..c5a3d5831--- /dev/null+++ b/internal/application/repository/retriever/sqlite/batch_update_test.go@@ -0,0 +1,222 @@+package sqlite++import (+ "bytes"+ "context"+ "fmt"+ "os"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// BatchUpdateChunkEnabledStatus / BatchUpdateChunkTagID only touch the index+// copy (lite_embeddings) that retrieval reads, while the authoritative chunks+// row is written by the service layer beforehand. Swallowing an UPDATE error+// therefore makes the call look successful while retrieval keeps filtering on+// the stale is_enabled / tag_id value, so the error has to reach the caller.++func TestBatchUpdateChunkEnabledStatusReturnsUpdateError(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex("chunk-1", "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )+ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings").Error)++ err := repository.BatchUpdateChunkEnabledStatus(context.Background(), map[string]bool{"chunk-1": false})++ require.Error(t, err)+ assert.Contains(t, err.Error(), "chunk-1")+ assert.Contains(t, err.Error(), "is_enabled")+}++func TestBatchUpdateChunkTagIDReturnsUpdateError(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex("chunk-1", "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )+ require.NoError(t, repository.db.Exec("DROP TABLE lite_embeddings").Error)++ err := repository.BatchUpdateChunkTagID(context.Background(), map[string]string{"chunk-1": "tag-2"})++ require.Error(t, err)+ assert.Contains(t, err.Error(), "chunk-1")+ assert.Contains(t, err.Error(), "tag_id")+}++// A statement that fails inside a batch must not be masked by the remaining+// statements of the same batch.+func TestBatchUpdateChunkTagIDReportsFailingStatementInBatch(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ for _, chunkID := range []string{"chunk-ok", "chunk-fail"} {+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex(chunkID, "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )+ }+ require.NoError(t, repository.db.Exec(`CREATE TRIGGER fail_chunk_tag_update+ BEFORE UPDATE ON lite_embeddings+ WHEN OLD.chunk_id = 'chunk-fail'+ BEGIN SELECT RAISE(ABORT, 'simulated index write failure'); END`).Error)++ err := repository.BatchUpdateChunkTagID(context.Background(), map[string]string{+ "chunk-ok": "tag-2",+ "chunk-fail": "tag-2",+ })++ require.Error(t, err)+ assert.Contains(t, err.Error(), "chunk-fail")+ assert.Contains(t, err.Error(), "simulated index write failure")+}++// Every failing statement of a batch has to be reported, not just the one that+// happened to be visited first: stopping at the first failure leaves an+// arbitrary (map-iteration-order-dependent) prefix of the index copy updated+// while blaming a single chunk. Each collected error can only exist if its own+// UPDATE was attempted, so the count proves the whole batch was run.+func TestBatchUpdateChunkTagIDReportsEveryFailingStatementInBatch(t *testing.T) {+ const batchSize = 5+ repository := newSQLiteRetrieverTestRepository(t)+ chunkTagMap := make(map[string]string, batchSize)+ for i := 1; i <= batchSize; i++ {+ chunkID := fmt.Sprintf("chunk-%d", i)+ chunkTagMap[chunkID] = "tag-2"+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex(chunkID, "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )+ }+ require.NoError(t, repository.db.Exec(`CREATE TRIGGER fail_chunk_tag_update+ BEFORE UPDATE ON lite_embeddings+ BEGIN SELECT RAISE(ABORT, 'simulated index write failure'); END`).Error)++ err := repository.BatchUpdateChunkTagID(context.Background(), chunkTagMap)++ require.Error(t, err)+ for i := 1; i <= batchSize; i++ {+ assert.Contains(t, err.Error(), fmt.Sprintf("chunk-%d", i))+ }+ assert.Equal(t, batchSize, strings.Count(err.Error(), "simulated index write failure"))+}++// A failing statement must not cut the batch short: the other engines (Milvus,+// Qdrant) accumulate the error and finish the batch, so the rows the batch was+// asked to update are still written. The assertions are on the resulting rows+// rather than on a position in the batch because map iteration order is random.+func TestBatchUpdateChunkTagIDKeepsUpdatingAfterStatementFailure(t *testing.T) {+ const batchSize = 5+ repository := newSQLiteRetrieverTestRepository(t)+ for i := 1; i <= batchSize; i++ {+ chunkID := fmt.Sprintf("chunk-%d", i)+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex(chunkID, "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )+ }+ require.NoError(t, repository.db.Exec(`CREATE TRIGGER fail_chunk_tag_update+ BEFORE UPDATE ON lite_embeddings+ WHEN OLD.chunk_id = 'chunk-3'+ BEGIN SELECT RAISE(ABORT, 'simulated index write failure'); END`).Error)++ err := repository.BatchUpdateChunkTagID(context.Background(), map[string]string{+ "chunk-1": "tag-2",+ "chunk-2": "tag-2",+ "chunk-3": "tag-2",+ "chunk-4": "tag-2",+ "chunk-5": "tag-2",+ })++ require.Error(t, err)+ assert.Contains(t, err.Error(), "chunk-3")+ assert.Contains(t, err.Error(), "simulated index write failure")++ var rows []sqliteEmbedding+ require.NoError(t, repository.db.Order("chunk_id").Find(&rows).Error)+ require.Len(t, rows, batchSize)+ for _, row := range rows {+ if row.ChunkID == "chunk-3" {+ assert.Equal(t, "tag-1", row.TagID, "the failing statement must not be applied")+ continue+ }+ assert.Equal(t, "tag-2", row.TagID, "chunk %s was not updated after the failure", row.ChunkID)+ }+}++// Matching no row is not an error -- the index copy may legitimately lag behind+// the authoritative chunks table -- but it used to be completely invisible, so+// both batch updates warn with the table and the chunk they were meant for.+func TestBatchUpdateChunkWarnsWhenStatementMatchesNoRow(t *testing.T) {+ testCases := []struct {+ name string+ column string+ update func(repository *sqliteRepository, ctx context.Context) error+ }{+ {+ name: "is_enabled",+ column: "is_enabled",+ update: func(repository *sqliteRepository, ctx context.Context) error {+ return repository.BatchUpdateChunkEnabledStatus(ctx, map[string]bool{+ "chunk-1": false,+ "chunk-missing": false,+ })+ },+ },+ {+ name: "tag_id",+ column: "tag_id",+ update: func(repository *sqliteRepository, ctx context.Context) error {+ return repository.BatchUpdateChunkTagID(ctx, map[string]string{+ "chunk-1": "tag-2",+ "chunk-missing": "tag-2",+ })+ },+ },+ }++ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex("chunk-1", "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )++ var logs bytes.Buffer+ logger.SetOutput(&logs)+ t.Cleanup(func() { logger.SetOutput(os.Stdout) })++ require.NoError(t, testCase.update(repository, context.Background()))++ output := logs.String()+ t.Logf("captured log output:\n%s", output)+ assert.Contains(t, output, "lite_embeddings."+testCase.column)+ assert.Contains(t, output, "matched 0 rows for chunk chunk-missing")+ assert.NotContains(t, output, "matched 0 rows for chunk chunk-1")+ })+ }+}++// Happy path: the updates still apply and report success.+func TestBatchUpdateChunkStatusAndTagApply(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ saveSQLiteTestVector(t, repository,+ sqliteTestIndex("chunk-1", "kb-1", "knowledge-1", "tag-1", true),+ []float32{1, 0},+ )++ require.NoError(t, repository.BatchUpdateChunkEnabledStatus(+ context.Background(), map[string]bool{"chunk-1": false}))+ require.NoError(t, repository.BatchUpdateChunkTagID(+ context.Background(), map[string]string{"chunk-1": "tag-2"}))++ var row sqliteEmbedding+ require.NoError(t, repository.db.Where("chunk_id = ?", "chunk-1").First(&row).Error)+ require.NotNil(t, row.IsEnabled)+ assert.False(t, *row.IsEnabled)+ assert.Equal(t, "tag-2", row.TagID)+}diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.goindex b0177b19a..1973a8880 100644--- a/internal/application/repository/retriever/sqlite/repository.go+++ b/internal/application/repository/retriever/sqlite/repository.go@@ -2,6 +2,7 @@ package sqliteimport ("context"+ "errors""fmt""strings""sync"@@ -352,16 +353,51 @@ func copiedSourceID(sourceID, sourceChunkID, targetChunkID string) string {return uuid.New().String()}+// Both batch updates write one statement per chunk, so a failing statement only+// affects its own row. Errors are therefore accumulated and the whole batch is+// attempted (like the Milvus and Qdrant engines do) instead of stopping at the+// first failure and leaving an arbitrary prefix of the map applied, only to be+// reported as a blanket failure.func (r *sqliteRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {+ var updateErrs []errorfor chunkID, enabled := range chunkStatusMap {- r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("is_enabled", enabled)+ if err := r.updateChunkIndexColumn(ctx, "is_enabled", chunkID, enabled); err != nil {+ updateErrs = append(updateErrs,+ fmt.Errorf("[SQLite] failed to update is_enabled for chunk %s: %w", chunkID, err))+ }}- return nil+ return errors.Join(updateErrs...)}func (r *sqliteRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap map[string]string) error {+ var updateErrs []errorfor chunkID, tagID := range chunkTagMap {- r.db.WithContext(ctx).Model(&sqliteEmbedding{}).Where("chunk_id = ?", chunkID).Update("tag_id", tagID)+ if err := r.updateChunkIndexColumn(ctx, "tag_id", chunkID, tagID); err != nil {+ updateErrs = append(updateErrs,+ fmt.Errorf("[SQLite] failed to update tag_id for chunk %s: %w", chunkID, err))+ }+ }+ return errors.Join(updateErrs...)+}++// updateChunkIndexColumn applies a single per-chunk UPDATE to the retrieval+// index copy. A statement matching no row is not an error -- that row may never+// have reached the index -- but it would otherwise stay completely invisible,+// so it is warned about with the table and chunk it was meant for.+func (r *sqliteRepository) updateChunkIndexColumn(+ ctx context.Context, column, chunkID string, value any,+) error {+ result := r.db.WithContext(ctx).Model(&sqliteEmbedding{}).+ Where("chunk_id = ?", chunkID).+ Update(column, value)+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected == 0 {+ logger.GetLogger(ctx).Warnf(+ "[SQLite] update of %s.%s matched 0 rows for chunk %s",+ sqliteEmbedding{}.TableName(), column, chunkID,+ )}return nil}@@ -468,6 +504,7 @@ func (r *sqliteRepository) keywordsRetrieve(ctx context.Context, params types.ReRetrieverType: types.KeywordsRetrieverType,}}, nil}+func (r *sqliteRepository) vectorRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {if len(params.Embedding) == 0 {return nil, nil