RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: correct ingestion retry handling and enforce chunk permissions (#20658)
adwsaf753b4e45aa
diff --git a/internal/ingestion/service/redelivery_counter_test.go b/internal/ingestion/service/redelivery_counter_test.goindex 9f3f1b9c8f..6df3ce5665 100644--- a/internal/ingestion/service/redelivery_counter_test.go+++ b/internal/ingestion/service/redelivery_counter_test.go@@ -122,8 +122,11 @@ func TestRunTask_RedeliveryAfterIncompleteRunCountsOnce(t *testing.T) {// MarkCompleted) - the task row is left RUNNING, so the broker redelivers.applyResult(ingestor, docID, kbID)(context.Background(), nil)- // Redelivery of the still-RUNNING task: re-runs and completes.- if terminal := ingestor.runTask(context.Background(), rcMsg(taskID, docID, kbID)); !terminal {+ // Redelivery prepares the still-RUNNING task before re-running the pipeline.+ // Preparation must preserve the counts already included in the KB aggregate.+ msg := rcMsg(taskID, docID, kbID)+ ingestor.ingestionTaskSvc.PrepareValidatedRun(context.Background(), msg)+ if terminal := ingestor.runTask(context.Background(), msg); !terminal {t.Fatalf("expected the redelivery run to complete (terminal=true)")}diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 56b5e58508..c50afa2df8 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -517,27 +517,11 @@ func (s *ChunkService) Get(ctx context.Context, req *service.GetChunkRequest, usreturn nil, fmt.Errorf("chunk_id is required")}- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)+ kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, userID)if err != nil {- return nil, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return nil, fmt.Errorf("user has no accessible tenants")- }-- // Find the tenant that owns this dataset- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return nil, fmt.Errorf("user does not have access to this dataset")+ return nil, err}+ targetTenantID := kb.TenantID// Verify the document belongs to the dataset, mirroring Python's get_chunk// (DocumentService.query(id=document_id, kb_id=dataset_id)).@@ -696,6 +680,17 @@ func (s *ChunkService) getKnowledgebaseByID(ctx context.Context, datasetID strinreturn s.kbDAO.GetByID(ctx, dao.DB, datasetID)}+func (s *ChunkService) getAccessibleKnowledgebase(ctx context.Context, datasetID, userID string) (*entity.Knowledgebase, error) {+ if !s.accessible(ctx, datasetID, userID) {+ return nil, fmt.Errorf("user does not have access to this dataset")+ }+ kb, err := s.getKnowledgebaseByID(ctx, datasetID)+ if err != nil || kb == nil {+ return nil, fmt.Errorf("knowledge base not found")+ }+ return kb, nil+}+func (s *ChunkService) getDocumentsByIDs(ctx context.Context, docIDs []string) ([]*entity.Document, error) {if s.getDocumentsByIDsFunc != nil {return s.getDocumentsByIDsFunc(docIDs)@@ -781,15 +776,6 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,return nil, fmt.Errorf("doc_id is required")}- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return nil, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return nil, fmt.Errorf("user has no accessible tenants")- }-// Get document to find its tenantdocDAO := dao.NewDocumentDAO()doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)@@ -800,23 +786,11 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,return nil, fmt.Errorf("document not found")}- // Get knowledge base to find tenant- kb, err := s.kbDAO.GetByID(ctx, dao.DB, doc.KbID)- if err != nil || kb == nil {- return nil, fmt.Errorf("knowledge base not found")- }-- // Find which tenant this document belongs to- var targetTenantID string- for _, tenant := range tenants {- if tenant.TenantID == kb.TenantID {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return nil, fmt.Errorf("user does not have access to this document")+ kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, userID)+ if err != nil {+ return nil, err}+ targetTenantID := kb.TenantIDindexName := fmt.Sprintf("ragflow_%s", targetTenantID)@@ -1028,27 +1002,11 @@ func (s *ChunkService) SwitchChunks(ctx context.Context, userID, datasetID, docureturn fmt.Errorf("req is null")}- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)+ kb, err := s.getAccessibleKnowledgebase(ctx, datasetID, userID)if err != nil {- return fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return fmt.Errorf("user has no accessible tenants")- }-- // Find the tenant that owns this dataset- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, datasetID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return fmt.Errorf("user does not have access to this dataset")+ return err}+ targetTenantID := kb.TenantIDdocDAO := dao.NewDocumentDAO()doc, err := docDAO.GetByID(ctx, dao.DB, documentID)@@ -1086,27 +1044,11 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunkreturn fmt.Errorf("chunk_id is required")}- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)+ kb, err := s.getAccessibleKnowledgebase(ctx, req.DatasetID, userID)if err != nil {- return fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return fmt.Errorf("user has no accessible tenants")- }-- // Find the tenant that owns this dataset- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return fmt.Errorf("user does not have access to this dataset")+ return err}+ targetTenantID := kb.TenantID// Verify document belongs to datasetdocDAO := dao.NewDocumentDAO()@@ -1277,15 +1219,6 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChunreturn 0, fmt.Errorf("doc_id is required")}- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return 0, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return 0, fmt.Errorf("user has no accessible tenants")- }-// Verify document exists and belongs to a dataset (do this first to get doc.KbID)docDAO := dao.NewDocumentDAO()doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)@@ -1293,18 +1226,11 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChunreturn 0, fmt.Errorf("document not found")}- // Find the tenant that owns this document- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, doc.KbID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return 0, fmt.Errorf("user does not have access to this document")+ kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, userID)+ if err != nil {+ return 0, err}+ targetTenantID := kb.TenantIDindexName := fmt.Sprintf("ragflow_%s", targetTenantID)diff --git a/internal/service/chunk/chunk_permissions_test.go b/internal/service/chunk/chunk_permissions_test.gonew file mode 100644index 0000000000..76e089eef9--- /dev/null+++ b/internal/service/chunk/chunk_permissions_test.go@@ -0,0 +1,89 @@+package chunk++import (+ "testing"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+)++func TestChunkManagementDatasetPermissions(t *testing.T) {+ for _, tt := range []struct {+ name string+ userID string+ permission entity.TenantPermission+ allowed bool+ }{+ {"owner private", "owner", entity.TenantPermissionMe, true},+ {"member private", "member", entity.TenantPermissionMe, false},+ {"member team", "member", entity.TenantPermissionTeam, true},+ {"unrelated team", "other", entity.TenantPermissionTeam, false},+ } {+ t.Run(tt.name, func(t *testing.T) {+ db := setupChunkTestDB(t)+ pushChunkTestDB(t, db)+ insertChunkTestUserTenant(t, "member", "owner")+ insertChunkTestKB(t, "kb-1", "owner")+ insertChunkTestDoc(t, "doc-1", "kb-1")+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").+ Update("permission", string(tt.permission)).Error; err != nil {+ t.Fatalf("set KB permission: %v", err)+ }++ ctx := t.Context()+ for _, op := range []struct {+ name string+ run func(*ChunkService) error+ }{+ {"get", func(s *ChunkService) error {+ s.docEngine = &getChunkTestEngine{chunk: map[string]interface{}{+ "id": "chunk-1", "doc_id": "doc-1", "content_with_weight": "body",+ }}+ _, err := s.Get(ctx, &service.GetChunkRequest{+ DatasetID: "kb-1", DocumentID: "doc-1", ChunkID: "chunk-1",+ }, tt.userID)+ return err+ }},+ {"list", func(s *ChunkService) error {+ s.docEngine = &listChunksSearchEngine{}+ _, err := s.List(ctx, &service.ListChunksRequest{DatasetID: "kb-1", DocID: "doc-1"}, tt.userID)+ return err+ }},+ {"switch", func(s *ChunkService) error {+ s.docEngine = &updateChunkTestEngine{}+ return s.SwitchChunks(ctx, tt.userID, "kb-1", "doc-1", 0, []string{"chunk-1"})+ }},+ {"update", func(s *ChunkService) error {+ s.docEngine = &updateChunkTestEngine{existingChunk: map[string]interface{}{+ "id": "chunk-1", "doc_id": "doc-1", "content_with_weight": "body",+ }}+ return s.UpdateChunk(ctx, &service.UpdateChunkRequest{+ DatasetID: "kb-1", DocumentID: "doc-1", ChunkID: "chunk-1",+ }, tt.userID)+ }},+ {"remove", func(s *ChunkService) error {+ s.docEngine = &parseTestDocEngine{}+ _, err := s.RemoveChunks(ctx, &service.RemoveChunksRequest{DocID: "doc-1", DeleteAll: true}, tt.userID)+ return err+ }},+ } {+ t.Run(op.name, func(t *testing.T) {+ svc := &ChunkService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ documentDAO: dao.NewDocumentDAO(),+ markWikiDirtyFunc: func(string, string, string, []string) {},+ }+ err := op.run(svc)+ if tt.allowed {+ if err != nil {+ t.Fatalf("authorized operation failed: %v", err)+ }+ } else if err == nil || err.Error() != "user does not have access to this dataset" {+ t.Fatalf("unauthorized operation error = %v, want access denied", err)+ }+ })+ }+ })+ }+}diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex a4fddd8055..8db3287eca 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -1832,11 +1832,15 @@ func insertChunkTestUserTenant(t *testing.T, userID, tenantID string) {t.Helper()status := "1"+ role := "normal"+ if userID == tenantID {+ role = "owner"+ }userTenant := &entity.UserTenant{ID: userID + "-" + tenantID,UserID: userID,TenantID: tenantID,- Role: "owner",+ Role: role,InvitedBy: userID,Status: &status,}@@ -1854,7 +1858,7 @@ func insertChunkTestKB(t *testing.T, id, tenantID string) {TenantID: tenantID,Name: id,EmbdID: "embedding-model",- Permission: string(entity.TenantPermissionMe),+ Permission: string(entity.TenantPermissionTeam),CreatedBy: tenantID,ParserConfig: entity.JSONMap{},Status: &status,@@ -2294,7 +2298,7 @@ func TestSwitchChunksUpdatesDocEngineWithAvailableInt(t *testing.T) {TenantID: "tenant-1",Name: "dataset",EmbdID: "embed",- Permission: string(entity.TenantPermissionMe),+ Permission: string(entity.TenantPermissionTeam),CreatedBy: "user-1",ParserID: string(entity.ParserTypeNaive),ParserConfig: entity.JSONMap{},diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex f16648cb77..f21e921ddd 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -278,14 +278,14 @@ func (s *IngestionTaskService) TransitionTaskToRunning(ctx context.Context, task// PrepareValidatedRun performs the document and run-log initialization only// after ReloadAndValidateRunIdentity has accepted the task's captured binding.+// Committed chunk/token counts must survive redelivery so ApplyDocCounts rolls+// only the result's delta into the knowledge base aggregate.func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *entity.IngestionTask) {if task == nil {return}if err := s.documentDAO.UpdateByID(ctx, dao.DB, task.DocumentID, map[string]interface{}{"progress": float64(0),- "chunk_num": int64(0),- "token_num": int64(0),"process_duration": float64(0),"process_begin_at": time.Now(),}); err != nil {diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex a0b9f0042b..5ce16d2d55 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -445,7 +445,7 @@ func TestIngestionTaskServiceStartRunningTransitionsCreatedTask(t *testing.T) {}// TestPrepareValidatedRunResetsDocumentProgress verifies validation precedes-// document initialization and leaves the legacy progress_msg untouched.+// document initialization while preserving committed counts and progress_msg.func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -478,14 +478,14 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {if doc.Progress != 0 {t.Fatalf("progress = %f, want 0", doc.Progress)}- if doc.ChunkNum != 0 {- t.Fatalf("chunk_num = %d, want 0", doc.ChunkNum)+ if doc.ChunkNum != 10 {+ t.Fatalf("chunk_num = %d, want 10", doc.ChunkNum)}- if doc.TokenNum != 0 {- t.Fatalf("token_num = %d, want 0", doc.TokenNum)+ if doc.TokenNum != 100 {+ t.Fatalf("token_num = %d, want 100", doc.TokenNum)}if doc.ProgressMsg == nil || *doc.ProgressMsg != "partial" {- t.Fatalf("progress_msg = %v, want the legacy value unchanged", doc.ProgressMsg)+ t.Fatalf("progress_msg = %v, want partial", doc.ProgressMsg)}if doc.ProcessBeginAt == nil || doc.ProcessBeginAt.IsZero() {t.Fatal("process_begin_at not set")