Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: correct ingestion retry handling and enforce chunk permissions (#20658)

adwsaf753b4e45aa
diff --git a/internal/ingestion/service/redelivery_counter_test.go b/internal/ingestion/service/redelivery_counter_test.go
index 9f3f1b9c8f..6df3ce5665 100644
--- a/internal/ingestion/service/redelivery_counter_test.go
+++ b/internal/ingestion/service/redelivery_counter_test.go
@@ -122,8 +122,11 @@ func TestRunTask_RedeliveryAfterIncompleteRunCountsOnce(t *testing.T) {
// MarkCompleted) - the task row is left RUNNING, so the broker redelivers.
applyResult(ingestor, docID, kbID)(context.Background(), nil)
- // Redelivery of the still-RUNNING task: re-runs and completes.
- if terminal := ingestor.runTask(context.Background(), rcMsg(taskID, docID, kbID)); !terminal {
+ // Redelivery prepares the still-RUNNING task before re-running the pipeline.
+ // Preparation must preserve the counts already included in the KB aggregate.
+ msg := rcMsg(taskID, docID, kbID)
+ ingestor.ingestionTaskSvc.PrepareValidatedRun(context.Background(), msg)
+ if terminal := ingestor.runTask(context.Background(), msg); !terminal {
t.Fatalf("expected the redelivery run to complete (terminal=true)")
}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 56b5e58508..c50afa2df8 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -517,27 +517,11 @@ func (s *ChunkService) Get(ctx context.Context, req *service.GetChunkRequest, us
return nil, fmt.Errorf("chunk_id is required")
}
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
+ kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, userID)
if err != nil {
- return nil, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return nil, fmt.Errorf("user has no accessible tenants")
- }
-
- // Find the tenant that owns this dataset
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return nil, fmt.Errorf("user does not have access to this dataset")
+ return nil, err
}
+ targetTenantID := kb.TenantID
// Verify the document belongs to the dataset, mirroring Python's get_chunk
// (DocumentService.query(id=document_id, kb_id=dataset_id)).
@@ -696,6 +680,17 @@ func (s *ChunkService) getKnowledgebaseByID(ctx context.Context, datasetID strin
return s.kbDAO.GetByID(ctx, dao.DB, datasetID)
}
+func (s *ChunkService) getAccessibleKnowledgebase(ctx context.Context, datasetID, userID string) (*entity.Knowledgebase, error) {
+ if !s.accessible(ctx, datasetID, userID) {
+ return nil, fmt.Errorf("user does not have access to this dataset")
+ }
+ kb, err := s.getKnowledgebaseByID(ctx, datasetID)
+ if err != nil || kb == nil {
+ return nil, fmt.Errorf("knowledge base not found")
+ }
+ return kb, nil
+}
+
func (s *ChunkService) getDocumentsByIDs(ctx context.Context, docIDs []string) ([]*entity.Document, error) {
if s.getDocumentsByIDsFunc != nil {
return s.getDocumentsByIDsFunc(docIDs)
@@ -781,15 +776,6 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
return nil, fmt.Errorf("doc_id is required")
}
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return nil, fmt.Errorf("user has no accessible tenants")
- }
-
// Get document to find its tenant
docDAO := dao.NewDocumentDAO()
doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
@@ -800,23 +786,11 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
return nil, fmt.Errorf("document not found")
}
- // Get knowledge base to find tenant
- kb, err := s.kbDAO.GetByID(ctx, dao.DB, doc.KbID)
- if err != nil || kb == nil {
- return nil, fmt.Errorf("knowledge base not found")
- }
-
- // Find which tenant this document belongs to
- var targetTenantID string
- for _, tenant := range tenants {
- if tenant.TenantID == kb.TenantID {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return nil, fmt.Errorf("user does not have access to this document")
+ kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, userID)
+ if err != nil {
+ return nil, err
}
+ targetTenantID := kb.TenantID
indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
@@ -1028,27 +1002,11 @@ func (s *ChunkService) SwitchChunks(ctx context.Context, userID, datasetID, docu
return fmt.Errorf("req is null")
}
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
+ kb, err := s.getAccessibleKnowledgebase(ctx, datasetID, userID)
if err != nil {
- return fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return fmt.Errorf("user has no accessible tenants")
- }
-
- // Find the tenant that owns this dataset
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, datasetID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return fmt.Errorf("user does not have access to this dataset")
+ return err
}
+ targetTenantID := kb.TenantID
docDAO := dao.NewDocumentDAO()
doc, err := docDAO.GetByID(ctx, dao.DB, documentID)
@@ -1086,27 +1044,11 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk
return fmt.Errorf("chunk_id is required")
}
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
+ kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, userID)
if err != nil {
- return fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return fmt.Errorf("user has no accessible tenants")
- }
-
- // Find the tenant that owns this dataset
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return fmt.Errorf("user does not have access to this dataset")
+ return err
}
+ targetTenantID := kb.TenantID
// Verify document belongs to dataset
docDAO := dao.NewDocumentDAO()
@@ -1277,15 +1219,6 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChun
return 0, fmt.Errorf("doc_id is required")
}
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return 0, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return 0, fmt.Errorf("user has no accessible tenants")
- }
-
// Verify document exists and belongs to a dataset (do this first to get doc.KbID)
docDAO := dao.NewDocumentDAO()
doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
@@ -1293,18 +1226,11 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChun
return 0, fmt.Errorf("document not found")
}
- // Find the tenant that owns this document
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, doc.KbID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return 0, fmt.Errorf("user does not have access to this document")
+ kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, userID)
+ if err != nil {
+ return 0, err
}
+ targetTenantID := kb.TenantID
indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
diff --git a/internal/service/chunk/chunk_permissions_test.go b/internal/service/chunk/chunk_permissions_test.go
new file mode 100644
index 0000000000..76e089eef9
--- /dev/null
+++ b/internal/service/chunk/chunk_permissions_test.go
@@ -0,0 +1,89 @@
+package chunk
+
+import (
+ "testing"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/service"
+)
+
+func TestChunkManagementDatasetPermissions(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ userID string
+ permission entity.TenantPermission
+ allowed bool
+ }{
+ {"owner private", "owner", entity.TenantPermissionMe, true},
+ {"member private", "member", entity.TenantPermissionMe, false},
+ {"member team", "member", entity.TenantPermissionTeam, true},
+ {"unrelated team", "other", entity.TenantPermissionTeam, false},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ db := setupChunkTestDB(t)
+ pushChunkTestDB(t, db)
+ insertChunkTestUserTenant(t, "member", "owner")
+ insertChunkTestKB(t, "kb-1", "owner")
+ insertChunkTestDoc(t, "doc-1", "kb-1")
+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").
+ Update("permission", string(tt.permission)).Error; err != nil {
+ t.Fatalf("set KB permission: %v", err)
+ }
+
+ ctx := t.Context()
+ for _, op := range []struct {
+ name string
+ run func(*ChunkService) error
+ }{
+ {"get", func(s *ChunkService) error {
+ s.docEngine = &getChunkTestEngine{chunk: map[string]interface{}{
+ "id": "chunk-1", "doc_id": "doc-1", "content_with_weight": "body",
+ }}
+ _, err := s.Get(ctx, &service.GetChunkRequest{
+ DatasetID: "kb-1", DocumentID: "doc-1", ChunkID: "chunk-1",
+ }, tt.userID)
+ return err
+ }},
+ {"list", func(s *ChunkService) error {
+ s.docEngine = &listChunksSearchEngine{}
+ _, err := s.List(ctx, &service.ListChunksRequest{DatasetID: "kb-1", DocID: "doc-1"}, tt.userID)
+ return err
+ }},
+ {"switch", func(s *ChunkService) error {
+ s.docEngine = &updateChunkTestEngine{}
+ return s.SwitchChunks(ctx, tt.userID, "kb-1", "doc-1", 0, []string{"chunk-1"})
+ }},
+ {"update", func(s *ChunkService) error {
+ s.docEngine = &updateChunkTestEngine{existingChunk: map[string]interface{}{
+ "id": "chunk-1", "doc_id": "doc-1", "content_with_weight": "body",
+ }}
+ return s.UpdateChunk(ctx, &service.UpdateChunkRequest{
+ DatasetID: "kb-1", DocumentID: "doc-1", ChunkID: "chunk-1",
+ }, tt.userID)
+ }},
+ {"remove", func(s *ChunkService) error {
+ s.docEngine = &parseTestDocEngine{}
+ _, err := s.RemoveChunks(ctx, &service.RemoveChunksRequest{DocID: "doc-1", DeleteAll: true}, tt.userID)
+ return err
+ }},
+ } {
+ t.Run(op.name, func(t *testing.T) {
+ svc := &ChunkService{
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ documentDAO: dao.NewDocumentDAO(),
+ markWikiDirtyFunc: func(string, string, string, []string) {},
+ }
+ err := op.run(svc)
+ if tt.allowed {
+ if err != nil {
+ t.Fatalf("authorized operation failed: %v", err)
+ }
+ } else if err == nil || err.Error() != "user does not have access to this dataset" {
+ t.Fatalf("unauthorized operation error = %v, want access denied", err)
+ }
+ })
+ }
+ })
+ }
+}
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index a4fddd8055..8db3287eca 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -1832,11 +1832,15 @@ func insertChunkTestUserTenant(t *testing.T, userID, tenantID string) {
t.Helper()
status := "1"
+ role := "normal"
+ if userID == tenantID {
+ role = "owner"
+ }
userTenant := &entity.UserTenant{
ID: userID + "-" + tenantID,
UserID: userID,
TenantID: tenantID,
- Role: "owner",
+ Role: role,
InvitedBy: userID,
Status: &status,
}
@@ -1854,7 +1858,7 @@ func insertChunkTestKB(t *testing.T, id, tenantID string) {
TenantID: tenantID,
Name: id,
EmbdID: "embedding-model",
- Permission: string(entity.TenantPermissionMe),
+ Permission: string(entity.TenantPermissionTeam),
CreatedBy: tenantID,
ParserConfig: entity.JSONMap{},
Status: &status,
@@ -2294,7 +2298,7 @@ func TestSwitchChunksUpdatesDocEngineWithAvailableInt(t *testing.T) {
TenantID: "tenant-1",
Name: "dataset",
EmbdID: "embed",
- Permission: string(entity.TenantPermissionMe),
+ Permission: string(entity.TenantPermissionTeam),
CreatedBy: "user-1",
ParserID: string(entity.ParserTypeNaive),
ParserConfig: entity.JSONMap{},
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index f16648cb77..f21e921ddd 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -278,14 +278,14 @@ func (s *IngestionTaskService) TransitionTaskToRunning(ctx context.Context, task
// PrepareValidatedRun performs the document and run-log initialization only
// after ReloadAndValidateRunIdentity has accepted the task's captured binding.
+// Committed chunk/token counts must survive redelivery so ApplyDocCounts rolls
+// only the result's delta into the knowledge base aggregate.
func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *entity.IngestionTask) {
if task == nil {
return
}
if err := s.documentDAO.UpdateByID(ctx, dao.DB, task.DocumentID, map[string]interface{}{
"progress": float64(0),
- "chunk_num": int64(0),
- "token_num": int64(0),
"process_duration": float64(0),
"process_begin_at": time.Now(),
}); err != nil {
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index a0b9f0042b..5ce16d2d55 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -445,7 +445,7 @@ func TestIngestionTaskServiceStartRunningTransitionsCreatedTask(t *testing.T) {
}
// TestPrepareValidatedRunResetsDocumentProgress verifies validation precedes
-// document initialization and leaves the legacy progress_msg untouched.
+// document initialization while preserving committed counts and progress_msg.
func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -478,14 +478,14 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {
if doc.Progress != 0 {
t.Fatalf("progress = %f, want 0", doc.Progress)
}
- if doc.ChunkNum != 0 {
- t.Fatalf("chunk_num = %d, want 0", doc.ChunkNum)
+ if doc.ChunkNum != 10 {
+ t.Fatalf("chunk_num = %d, want 10", doc.ChunkNum)
}
- if doc.TokenNum != 0 {
- t.Fatalf("token_num = %d, want 0", doc.TokenNum)
+ if doc.TokenNum != 100 {
+ t.Fatalf("token_num = %d, want 100", doc.TokenNum)
}
if doc.ProgressMsg == nil || *doc.ProgressMsg != "partial" {
- t.Fatalf("progress_msg = %v, want the legacy value unchanged", doc.ProgressMsg)
+ t.Fatalf("progress_msg = %v, want partial", doc.ProgressMsg)
}
if doc.ProcessBeginAt == nil || doc.ProcessBeginAt.IsZero() {
t.Fatal("process_begin_at not set")