RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: 1. update document parser_config does not clean chunks, 2. after update, pop up dialog to notice use to re-parse (#21281)
Wang Qifeab371067f7
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.mdindex 151cc03207..a3706c70f6 100644--- a/docs/references/http_api_reference.md+++ b/docs/references/http_api_reference.md@@ -1202,6 +1202,8 @@ Failure:Updates configurations for a specified document.+Saving parser settings preserves existing chunks, chunk images, counters, and parsing progress. The saved settings take effect on the next parse; this update does not start parsing or delete existing results. Use the document parse action to clear existing chunks and re-parse with the saved settings.+#### Request- Method: PATCHdiff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 6bf1e561bf..013235e6aa 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -154,7 +154,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat// object. sel is the request selection (nil when the whole selection block// was omitted); current is the dataset's persisted selection. Resolve applies// PATCH semantics: an omitted block inherits the current mode. Both the- // parser_config cleaning and the reparse targeting derive from this single+ // parser_config cleaning and the parser selection derive from this single// resolution so they can never disagree.sel, selErr := service.FromRequest(req.ParseType, req.ParserID, req.PipelineID)if selErr != nil {@@ -205,41 +205,36 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat}}- // Apply parser_id / pipeline_id changes. parse_type is validated earlier- // (in validateDatasetDocumentUpdate); reparse only fires when parse_type- // explicitly selects a mode. The mode direction comes from the same- // isPipeline resolution above.- var reparseParserID, reparsePipelineID *string+ // Save the selection without changing existing parse results. Cleanup is+ // owned by the explicit reparse request.+ parserUpdates := map[string]interface{}{}if req.ParseType != nil {switch {case !isPipeline: // BuiltInif present["parser_id"] && req.ParserID != nil {if p := strings.TrimSpace(*req.ParserID); p != "" {- reparseParserID = &p+ parserUpdates["parser_id"] = p}}// Drop any prior canvas so the worker falls back to the builtin template.- empty := ""- reparsePipelineID = &empty+ parserUpdates["pipeline_id"] = ""case isPipeline: // Pipelineif present["pipeline_id"] && req.PipelineID != nil {- reparsePipelineID = req.PipelineID+ parserUpdates["pipeline_id"] = *req.PipelineID}}} else if present["chunk_method"] && req.ChunkMethod != nil {// chunk_method is the public alias for a direct parser change; it does// not require parse_type (mirrors Python's update_chunk_method).if p := strings.TrimSpace(*req.ChunkMethod); !strings.EqualFold(p, doc.ParserID) {- reparseParserID = &p- empty := ""- reparsePipelineID = &empty+ parserUpdates["parser_id"] = p+ parserUpdates["pipeline_id"] = ""} else if doc.PipelineID != nil && *doc.PipelineID != "" {- empty := ""- reparsePipelineID = &empty+ parserUpdates["pipeline_id"] = ""}}- if reparseParserID != nil || reparsePipelineID != nil {- if err = s.resetDocumentForReparse(ctx, doc, kb.TenantID, reparseParserID, reparsePipelineID); err != nil {+ if len(parserUpdates) > 0 {+ if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, parserUpdates); err != nil {return nil, common.CodeDataError, err}}diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex 002d1bf080..a6469edc95 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -428,48 +428,6 @@ func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Documereturn nil}-func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *entity.Document, tenantID string, parserID *string, pipelineID *string) error {- updates := map[string]interface{}{- "progress": 0,- }- if parserID != nil {- updates["parser_id"] = *parserID- }- if pipelineID != nil {- updates["pipeline_id"] = *pipelineID- }-- if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, updates); err != nil {- return errors.New("document not found")- }-- if doc.TokenNum > 0 {- decremented, err := s.decrementDocumentAndKBCountersForReparse(doc)- if err != nil {- return errors.New("document not found")- }- if !decremented {- return nil- }- if s.docEngine != nil {- indexName := fmt.Sprintf("ragflow_%s", tenantID)- s.deleteChunkImages(ctx, doc, indexName)- if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {- return err- }- }- }-- return nil-}--func (s *DocumentService) deleteChunkImages(ctx context.Context, doc *entity.Document, indexName string) {- if doc == nil {- return- }- _ = s.deleteDocumentChunkImages(ctx, indexName, doc.KbID, doc.ID)-}-// deleteDocumentChunkImages removes source chunk image objects in bounded// search batches. Repeating the operation is safe because deleting an absent// object is a no-op.@@ -549,34 +507,6 @@ func firstStringField(m map[string]interface{}, keys ...string) string {return ""}-func (s *DocumentService) decrementDocumentAndKBCountersForReparse(doc *entity.Document) (bool, error) {- decremented := false- err := dao.DB.Transaction(func(tx *gorm.DB) error {- result := tx.Model(&entity.Document{}).- Where("id = ? AND kb_id = ? AND token_num = ? AND chunk_num = ?", doc.ID, doc.KbID, doc.TokenNum, doc.ChunkNum).- Updates(map[string]interface{}{- "token_num": gorm.Expr("token_num - ?", doc.TokenNum),- "chunk_num": gorm.Expr("chunk_num - ?", doc.ChunkNum),- "process_duration": gorm.Expr("process_duration - ?", doc.ProcessDuration),- })- if result.Error != nil {- return result.Error- }- if result.RowsAffected == 0 {- return nil- }- decremented = true-- return tx.Model(&entity.Knowledgebase{}).- Where("id = ?", doc.KbID).- Updates(map[string]interface{}{- "token_num": gorm.Expr("token_num - ?", doc.TokenNum),- "chunk_num": gorm.Expr("chunk_num - ?", doc.ChunkNum),- }).Error- })- return decremented, err-}-func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, status int) error {statusStr := strconv.Itoa(status)if doc.Status != nil && *doc.Status == statusStr {diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 9c53001f87..6b9c283f36 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -2540,68 +2540,86 @@ func TestUpdateDatasetDocumentRenameUpdatesDocumentAndFile(t *testing.T) {}}-func TestUpdateDatasetDocumentParserIDResetsForReparse(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)-- parseType := 1- chunkMethod := "manual"- svc := testDocumentService(t)- ctx := t.Context()- resp, code, err := svc.UpdateDatasetDocument(ctx, "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{- ParserID: &chunkMethod,- ParseType: &parseType,- }, map[string]bool{"parser_id": true, "parse_type": true})- if err != nil {- t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)- }- if resp.ParserID != chunkMethod || resp.IngestionStatus != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {- t.Fatalf("response = %+v, want method=%s ingestion_status=UNSTART counts=0", resp, chunkMethod)- }-- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")- if doc.ParserID != chunkMethod {- t.Fatalf("parser_id = %q, want %q", doc.ParserID, chunkMethod)- }- if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.Progress != 0 {- t.Fatalf("doc counters/progress = token:%d chunk:%d progress:%f, want zero", doc.TokenNum, doc.ChunkNum, doc.Progress)- }- kb, _ := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")- if kb.TokenNum != 0 || kb.ChunkNum != 0 {- t.Fatalf("kb counters = token:%d chunk:%d, want zero", kb.TokenNum, kb.ChunkNum)- }-}--func TestResetDocumentForReparseSkipsSecondCounterDecrement(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)-- ctx := t.Context()- staleDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")- if err != nil {- t.Fatalf("get doc: %v", err)- }-- svc := testDocumentService(t)- parserID := "manual"- if err = svc.resetDocumentForReparse(ctx, staleDoc, "tenant-1", &parserID, nil); err != nil {- t.Fatalf("first resetDocumentForReparse failed: %v", err)- }- if err = svc.resetDocumentForReparse(ctx, staleDoc, "tenant-1", &parserID, nil); err != nil {- t.Fatalf("second resetDocumentForReparse failed: %v", err)- }+func TestUpdateDatasetDocumentParserConfigPreservesParseResults(t *testing.T) {+ builtin, pipeline := 1, 2+ pipelineID := "1234567890abcdef1234567890abcdef"+ cases := []struct {+ name string+ request UpdateDatasetDocumentRequest+ present map[string]bool+ wantParser string+ wantPipeline string+ }{+ {name: "config only", wantParser: "naive"},+ {name: "same builtin", request: UpdateDatasetDocumentRequest{ParseType: &builtin, ParserID: sptr("naive")}, present: map[string]bool{"parse_type": true, "parser_id": true}, wantParser: "naive"},+ {name: "different builtin", request: UpdateDatasetDocumentRequest{ParseType: &builtin, ParserID: sptr("manual")}, present: map[string]bool{"parse_type": true, "parser_id": true}, wantParser: "manual"},+ {name: "pipeline", request: UpdateDatasetDocumentRequest{ParseType: &pipeline, PipelineID: &pipelineID}, present: map[string]bool{"parse_type": true, "pipeline_id": true}, wantParser: "naive", wantPipeline: pipelineID},+ {name: "chunk method", request: UpdateDatasetDocumentRequest{ChunkMethod: sptr("manual")}, present: map[string]bool{"chunk_method": true}, wantParser: "manual"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)+ insertTestCanvas(t, pipelineID, "user-1")+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_duration", 42).Error; err != nil {+ t.Fatalf("prepare document: %v", err)+ }- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")- if doc.TokenNum != 0 || doc.ChunkNum != 0 {- t.Fatalf("doc counters = token:%d chunk:%d, want zero", doc.TokenNum, doc.ChunkNum)- }- kb, _ := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")- if kb.TokenNum != 0 || kb.ChunkNum != 0 {- t.Fatalf("kb counters = token:%d chunk:%d, want zero after duplicate reset", kb.TokenNum, kb.ChunkNum)+ request := tc.request+ request.ParserConfig = map[string]any{+ "Extractor:AutoExtractDefault": map[string]any{"keywords": map[string]any{"top_n": 5}},+ }+ present := tc.present+ if present == nil {+ present = map[string]bool{}+ }+ present["parser_config"] = true+ engine := &rerunDeleteDocEngine{}+ svc := testDocumentService(t)+ svc.docEngine = engine+ resp, code, err := svc.UpdateDatasetDocument(t.Context(), "user-1", "kb-1", "doc-1", &request, present)+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)+ }+ if resp.ParserID != tc.wantParser || resp.IngestionStatus != string(common.COMPLETED) || resp.TokenCount != 10 || resp.ChunkCount != 5 || resp.Progress != 0.75 || resp.ProcessDuration != 42 {+ t.Fatalf("response changed parse results: %+v", resp)+ }+ if tc.wantPipeline != "" && (resp.PipelineID == nil || *resp.PipelineID != tc.wantPipeline) {+ t.Fatalf("pipeline_id = %v, want %q", resp.PipelineID, tc.wantPipeline)+ }+ doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("get document: %v", err)+ }+ if doc.TokenNum != 10 || doc.ChunkNum != 5 || doc.Progress != 0.75 || doc.ProcessDuration != 42 {+ t.Fatalf("persisted parse results changed: %+v", doc)+ }+ extractor, ok := doc.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})+ if !ok {+ t.Fatalf("extractor config not saved: %#v", doc.ParserConfig)+ }+ keywords, ok := extractor["keywords"].(map[string]interface{})+ if !ok || keywords["top_n"] != float64(5) {+ t.Fatalf("parser config not saved: %#v", doc.ParserConfig)+ }+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatalf("get dataset: %v", err)+ }+ if kb.TokenNum != 10 || kb.ChunkNum != 5 {+ t.Fatalf("dataset counters changed: token=%d chunk=%d", kb.TokenNum, kb.ChunkNum)+ }+ if engine.deleteCalls != 0 || engine.search != nil {+ t.Fatalf("existing chunks accessed during settings update: deletes=%d search=%+v", engine.deleteCalls, engine.search)+ }+ task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1")+ if err != nil || task == nil || task.ID != "task-1" || task.Status != common.COMPLETED {+ t.Fatalf("existing ingestion task changed: task=%+v err=%v", task, err)+ }+ })}}diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 6a7d219c46..93dbc41491 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -491,6 +491,8 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default limapplyAutoMetadataSettings: 'Apply global auto-metadata settings',parseFileTip: 'Are you sure to parse?',clearChunksReparseTip: 'Are you sure to clear chunks to re-parse?',+ parserConfigReparseTip:+ 'Existing chunks were generated using the previous parser settings. Would you like to clear them and re-parse this document using the saved settings?',parseFile: 'Parse file',emptyMetadata: 'No metadata',metadataField: 'Metadata field',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 2085d7d29e..4dbdeaae6c 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -438,6 +438,8 @@ export default {applyAutoMetadataSettings: '应用全局自动元数据设置',parseFileTip: '您确定要解析吗?',clearChunksReparseTip: '您确定要清空分块重新解析吗?',+ parserConfigReparseTip:+ '现有分块由之前的解析设置生成。是否清空现有分块,并使用已保存的设置重新解析此文档?',parseFile: '解析文件',emptyMetadata: '无元数据',localUpload: '本地上传',diff --git a/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx b/web/src/pages/dataset/dataset/change-parser-dialog.test.tsxindex e96ba58e02..d5767e42d0 100644--- a/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx+++ b/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx@@ -3,6 +3,7 @@ import { ChangeParserDialog } from './change-parser-dialog';import { useChangeDocumentParser } from './use-change-document-parser';const mockSave = jest.fn().mockResolvedValue(0);+const mockRun = jest.fn().mockResolvedValue(0);const mockDsl = {graph: {nodes: [@@ -40,6 +41,7 @@ jest.mock('@/hooks/use-agent-request', () => ({}),}));jest.mock('@/hooks/use-document-request', () => ({+ useRunDocument: () => ({ runDocumentByIds: mockRun, loading: false }),useSetDocumentPipelineParser: () => ({setDocumentPipelineParser: mockSave,loading: false,@@ -93,12 +95,19 @@ jest.mock('react-i18next', () => ({useTranslation: () => ({ t: (key: string) => key }),}));-function Harness({ builtin = false }: { builtin?: boolean }) {+function Harness({+ builtin = false,+ chunkCount = 0,+}: {+ builtin?: boolean;+ chunkCount?: number;+}) {const state = useChangeDocumentParser();const openDialog = () => {state.showChangeParserModal({id: 'document',dataset_id: 'dataset',+ chunk_count: chunkCount,chunk_method: builtin ? 'general' : '',pipeline_id: builtin ? '' : 'pipeline',parser_config: builtin@@ -128,10 +137,60 @@ function Harness({ builtin = false }: { builtin?: boolean }) {}beforeEach(() => {- mockSave.mockClear();+ mockSave.mockReset().mockResolvedValue(0);+ mockRun.mockReset().mockResolvedValue(0);mockCustomDsl = mockDsl;});+it('keeps existing chunks when re-parsing is declined after saving', async () => {+ render(<Harness builtin chunkCount={5} />);+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));+ expect(+ await screen.findByText('knowledgeDetails.parserConfigReparseTip'),+ ).toBeInTheDocument();+ expect(mockSave).toHaveBeenCalledTimes(1);+ expect(mockRun).not.toHaveBeenCalled();+ fireEvent.click(screen.getByRole('button', { name: 'common.no' }));+ await waitFor(() =>+ expect(+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),+ ).not.toBeInTheDocument(),+ );+ expect(mockRun).not.toHaveBeenCalled();+});++it('re-parses with the saved document settings only after confirmation', async () => {+ render(<Harness chunkCount={5} />);+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));+ const confirm = await screen.findByRole('button', { name: 'common.yes' });+ expect(mockSave).toHaveBeenCalledTimes(1);+ expect(mockRun).not.toHaveBeenCalled();+ fireEvent.click(confirm);+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));+ expect(mockRun).toHaveBeenCalledWith({+ documentIds: ['document'],+ run: 1,+ option: { delete: true, apply_kb: false },+ });+});++it('does not offer re-parsing when saving fails', async () => {+ mockSave.mockResolvedValueOnce(102);+ render(<Harness builtin chunkCount={5} />);+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));+ await waitFor(() => expect(mockSave).toHaveBeenCalledTimes(1));+ expect(+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),+ ).not.toBeInTheDocument();+ expect(+ screen.getByRole('button', { name: 'common.save' }),+ ).toBeInTheDocument();+ expect(mockRun).not.toHaveBeenCalled();+});+it('saves the built-in parser', async () => {render(<Harness builtin />);fireEvent.click(screen.getByRole('button', { name: 'Open' }));@@ -144,6 +203,10 @@ it('saves the built-in parser', async () => {parse_type: 1,}),);+ expect(+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),+ ).not.toBeInTheDocument();+ expect(mockRun).not.toHaveBeenCalled();});it('saves after switching from a built-in parser to a pipeline', async () => {diff --git a/web/src/pages/dataset/dataset/use-change-document-parser.ts b/web/src/pages/dataset/dataset/use-change-document-parser.tsindex 8d6a5b1ce8..267a43e475 100644--- a/web/src/pages/dataset/dataset/use-change-document-parser.ts+++ b/web/src/pages/dataset/dataset/use-change-document-parser.ts@@ -1,11 +1,18 @@+import { Modal } from '@/components/ui/modal/modal';import { useSetModalState } from '@/hooks/common-hooks';-import { useSetDocumentPipelineParser } from '@/hooks/use-document-request';+import {+ useRunDocument,+ useSetDocumentPipelineParser,+} from '@/hooks/use-document-request';import { IDocumentInfo } from '@/interfaces/database/document';import { IChangeParserRequestBody } from '@/interfaces/request/document';import { useCallback, useState } from 'react';+import { useTranslation } from 'react-i18next';export const useChangeDocumentParser = () => {+ const { t } = useTranslation();const { setDocumentPipelineParser, loading } = useSetDocumentPipelineParser();+ const { runDocumentByIds } = useRunDocument();const [record, setRecord] = useState<IDocumentInfo>({} as IDocumentInfo);const {@@ -29,14 +36,36 @@ export const useChangeDocumentParser = () => {});if (ret === 0) {hideChangeParserModal();+ if (record.chunk_count > 0) {+ Modal.confirm({+ title: t('knowledgeDetails.parseFile'),+ content: t('knowledgeDetails.parserConfigReparseTip'),+ okText: t('common.yes'),+ cancelText: t('common.no'),+ onOk: async () => {+ try {+ await runDocumentByIds({+ documentIds: [record.id],+ run: 1,+ option: { delete: true, apply_kb: false },+ });+ } catch (error) {+ console.warn(error);+ }+ },+ });+ }}}},[record?.id,record?.dataset_id,+ record?.chunk_count,+ runDocumentByIds,setDocumentPipelineParser,hideChangeParserModal,+ t,],);