Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): delete written chunks when the storage quota check fails (#3656)
Jincbfd66fd922d
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 0329ba91b..ee9760467 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -339,6 +339,16 @@ func buildParentChildConfigs(cc types.ChunkingConfig, base chunker.SplitterConfireturn chunker.DeriveParentChildConfigs(base, cc.ParentChunkSize, cc.ChildChunkSize)}+// deleteUnindexedChunks drops the chunks processChunks wrote for a knowledge+// that failed before BatchIndex ran. Nothing reached the vector store yet, so+// only the chunk rows need removing; left alone they would stay active under a+// failed knowledge, the same leftovers the BatchIndex failure path cleans up.+func (s *knowledgeService) deleteUnindexedChunks(ctx context.Context, knowledge *types.Knowledge) {+ if err := s.chunkRepo.DeleteChunksByKnowledgeID(ctx, knowledge.TenantID, knowledge.ID); err != nil {+ logger.Errorf(ctx, "Delete chunks failed: %v", err)+ }+}+// processChunks processes chunks and creates embeddings for knowledge contentfunc (s *knowledgeService) processChunks(ctx context.Context,kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []types.ParsedChunk,@@ -693,6 +703,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,knowledge.ErrorMessage = err.Error()knowledge.UpdatedAt = time.Now()s.repo.UpdateKnowledge(ctx, knowledge)+ s.deleteUnindexedChunks(ctx, knowledge)return nil}// Check if there's enough storage quota available@@ -701,6 +712,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,knowledge.ErrorMessage = "存储空间不足"knowledge.UpdatedAt = time.Now()s.repo.UpdateKnowledge(ctx, knowledge)+ s.deleteUnindexedChunks(ctx, knowledge)return nil}}diff --git a/internal/application/service/knowledge_process_quota_test.go b/internal/application/service/knowledge_process_quota_test.gonew file mode 100644index 000000000..5b3009577--- /dev/null+++ b/internal/application/service/knowledge_process_quota_test.go@@ -0,0 +1,122 @@+package service++import (+ "context"+ "errors"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/embedding"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++// quotaChunkRepo records whether chunks were deleted after processChunks+// wrote them; the delete at the start of processChunks is the idempotent+// cleanup of a previous attempt and does not count.+type quotaChunkRepo struct {+ interfaces.ChunkRepository+ created bool+ deletesAfterCreated int+}++func (r *quotaChunkRepo) DeleteChunksByKnowledgeID(context.Context, uint64, string) error {+ if r.created {+ r.deletesAfterCreated+++ }+ return nil+}++func (r *quotaChunkRepo) CreateChunks(context.Context, []*types.Chunk) error {+ r.created = true+ return nil+}++type quotaRetrieveEngine struct {+ parentChildRetrieveEngine+ estimate int64+}++func (e *quotaRetrieveEngine) EstimateStorageSize(+ context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType,+) int64 {+ return e.estimate+}++type quotaTenantRepo struct {+ parentChildTenantRepo+ tenant *types.Tenant+ err error+}++func (r quotaTenantRepo) GetTenantByID(context.Context, uint64) (*types.Tenant, error) {+ return r.tenant, r.err+}++// A knowledge that fails the storage quota check (or cannot re-read the+// tenant for it) must not leave the chunks written just before the check+// active under a failed knowledge — the BatchIndex failure path already+// removes them, and the quota branches have to do the same.+func TestProcessChunksDeletesChunksWhenStorageQuotaCheckFails(t *testing.T) {+ for _, tc := range []struct {+ name string+ tenant *types.Tenant+ tenantErr error+ }{+ {+ name: "quota exceeded",+ tenant: &types.Tenant{ID: 1, StorageQuota: 100, StorageUsed: 90},+ },+ {+ name: "tenant re-read fails",+ tenantErr: errors.New("tenant lookup failed"),+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ knowledge := &types.Knowledge{+ ID: "knowledge-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ ParseStatus: types.ParseStatusProcessing,+ }+ chunkRepo := "aChunkRepo{}+ retrieveEngine := "aRetrieveEngine{estimate: 50}+ tenant := &types.Tenant{+ ID: 1,+ StorageQuota: 100,+ RetrieverEngines: types.RetrieverEngines{Engines: []types.RetrieverEngineParams{+ {+ RetrieverType: types.VectorRetrieverType,+ RetrieverEngineType: types.PostgresRetrieverEngineType,+ },+ }},+ }+ ctx := context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)+ svc := &knowledgeService{+ repo: &parentChildKnowledgeRepo{knowledge: knowledge},+ chunkRepo: chunkRepo,+ modelService: parentChildModelService{embedder: parentChildEmbedder{}},+ retrieveEngine: parentChildRetrieveRegistry{engine: retrieveEngine},+ graphEngine: parentChildGraphRepo{},+ tenantRepo: quotaTenantRepo{tenant: tc.tenant, err: tc.tenantErr},+ task: parentChildTaskEnqueuer{},+ }+ kb := &types.KnowledgeBase{+ ID: "kb-1",+ TenantID: 1,+ EmbeddingModelID: "embedding-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true},+ }++ // Terminal for this attempt: the failure is recorded, nothing to retry.+ require.NoError(t, svc.processChunks(ctx, kb, knowledge,+ []types.ParsedChunk{{Content: "body", Seq: 0, Start: 0, End: 4}}))++ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)+ require.True(t, chunkRepo.created, "the quota check runs after the chunks are written")+ require.Equal(t, 1, chunkRepo.deletesAfterCreated,+ "the chunks written for this attempt must be deleted when it fails")+ require.Empty(t, retrieveEngine.indexed, "a failed quota check must not index anything")+ })+ }+}