Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: 1. update document parser_config does not clean chunks, 2. after update, pop up dialog to notice use to re-parse (#21281)

Wang Qifeab371067f7
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md
index 151cc03207..a3706c70f6 100644
--- a/docs/references/http_api_reference.md
+++ b/docs/references/http_api_reference.md
@@ -1202,6 +1202,8 @@ Failure:
Updates configurations for a specified document.
+Saving parser settings preserves existing chunks, chunk images, counters, and parsing progress. The saved settings take effect on the next parse; this update does not start parsing or delete existing results. Use the document parse action to clear existing chunks and re-parse with the saved settings.
+
#### Request
- Method: PATCH
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 6bf1e561bf..013235e6aa 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -154,7 +154,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
// object. sel is the request selection (nil when the whole selection block
// was omitted); current is the dataset's persisted selection. Resolve applies
// PATCH semantics: an omitted block inherits the current mode. Both the
- // parser_config cleaning and the reparse targeting derive from this single
+ // parser_config cleaning and the parser selection derive from this single
// resolution so they can never disagree.
sel, selErr := service.FromRequest(req.ParseType, req.ParserID, req.PipelineID)
if selErr != nil {
@@ -205,41 +205,36 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
}
}
- // Apply parser_id / pipeline_id changes. parse_type is validated earlier
- // (in validateDatasetDocumentUpdate); reparse only fires when parse_type
- // explicitly selects a mode. The mode direction comes from the same
- // isPipeline resolution above.
- var reparseParserID, reparsePipelineID *string
+ // Save the selection without changing existing parse results. Cleanup is
+ // owned by the explicit reparse request.
+ parserUpdates := map[string]interface{}{}
if req.ParseType != nil {
switch {
case !isPipeline: // BuiltIn
if present["parser_id"] && req.ParserID != nil {
if p := strings.TrimSpace(*req.ParserID); p != "" {
- reparseParserID = &p
+ parserUpdates["parser_id"] = p
}
}
// Drop any prior canvas so the worker falls back to the builtin template.
- empty := ""
- reparsePipelineID = &empty
+ parserUpdates["pipeline_id"] = ""
case isPipeline: // Pipeline
if present["pipeline_id"] && req.PipelineID != nil {
- reparsePipelineID = req.PipelineID
+ parserUpdates["pipeline_id"] = *req.PipelineID
}
}
} else if present["chunk_method"] && req.ChunkMethod != nil {
// chunk_method is the public alias for a direct parser change; it does
// not require parse_type (mirrors Python's update_chunk_method).
if p := strings.TrimSpace(*req.ChunkMethod); !strings.EqualFold(p, doc.ParserID) {
- reparseParserID = &p
- empty := ""
- reparsePipelineID = &empty
+ parserUpdates["parser_id"] = p
+ parserUpdates["pipeline_id"] = ""
} else if doc.PipelineID != nil && *doc.PipelineID != "" {
- empty := ""
- reparsePipelineID = &empty
+ parserUpdates["pipeline_id"] = ""
}
}
- if reparseParserID != nil || reparsePipelineID != nil {
- if err = s.resetDocumentForReparse(ctx, doc, kb.TenantID, reparseParserID, reparsePipelineID); err != nil {
+ if len(parserUpdates) > 0 {
+ if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, parserUpdates); err != nil {
return nil, common.CodeDataError, err
}
}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 002d1bf080..a6469edc95 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -428,48 +428,6 @@ func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Docume
return nil
}
-func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *entity.Document, tenantID string, parserID *string, pipelineID *string) error {
- updates := map[string]interface{}{
- "progress": 0,
- }
- if parserID != nil {
- updates["parser_id"] = *parserID
- }
- if pipelineID != nil {
- updates["pipeline_id"] = *pipelineID
- }
-
- if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, updates); err != nil {
- return errors.New("document not found")
- }
-
- if doc.TokenNum > 0 {
- decremented, err := s.decrementDocumentAndKBCountersForReparse(doc)
- if err != nil {
- return errors.New("document not found")
- }
- if !decremented {
- return nil
- }
- if s.docEngine != nil {
- indexName := fmt.Sprintf("ragflow_%s", tenantID)
- s.deleteChunkImages(ctx, doc, indexName)
- if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {
- return err
- }
- }
- }
-
- return nil
-}
-
-func (s *DocumentService) deleteChunkImages(ctx context.Context, doc *entity.Document, indexName string) {
- if doc == nil {
- return
- }
- _ = s.deleteDocumentChunkImages(ctx, indexName, doc.KbID, doc.ID)
-}
-
// deleteDocumentChunkImages removes source chunk image objects in bounded
// search batches. Repeating the operation is safe because deleting an absent
// object is a no-op.
@@ -549,34 +507,6 @@ func firstStringField(m map[string]interface{}, keys ...string) string {
return ""
}
-func (s *DocumentService) decrementDocumentAndKBCountersForReparse(doc *entity.Document) (bool, error) {
- decremented := false
- err := dao.DB.Transaction(func(tx *gorm.DB) error {
- result := tx.Model(&entity.Document{}).
- Where("id = ? AND kb_id = ? AND token_num = ? AND chunk_num = ?", doc.ID, doc.KbID, doc.TokenNum, doc.ChunkNum).
- Updates(map[string]interface{}{
- "token_num": gorm.Expr("token_num - ?", doc.TokenNum),
- "chunk_num": gorm.Expr("chunk_num - ?", doc.ChunkNum),
- "process_duration": gorm.Expr("process_duration - ?", doc.ProcessDuration),
- })
- if result.Error != nil {
- return result.Error
- }
- if result.RowsAffected == 0 {
- return nil
- }
- decremented = true
-
- return tx.Model(&entity.Knowledgebase{}).
- Where("id = ?", doc.KbID).
- Updates(map[string]interface{}{
- "token_num": gorm.Expr("token_num - ?", doc.TokenNum),
- "chunk_num": gorm.Expr("chunk_num - ?", doc.ChunkNum),
- }).Error
- })
- return decremented, err
-}
-
func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, status int) error {
statusStr := strconv.Itoa(status)
if doc.Status != nil && *doc.Status == statusStr {
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 9c53001f87..6b9c283f36 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -2540,68 +2540,86 @@ func TestUpdateDatasetDocumentRenameUpdatesDocumentAndFile(t *testing.T) {
}
}
-func TestUpdateDatasetDocumentParserIDResetsForReparse(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
- insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
-
- parseType := 1
- chunkMethod := "manual"
- svc := testDocumentService(t)
- ctx := t.Context()
- resp, code, err := svc.UpdateDatasetDocument(ctx, "user-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
- ParserID: &chunkMethod,
- ParseType: &parseType,
- }, map[string]bool{"parser_id": true, "parse_type": true})
- if err != nil {
- t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)
- }
- if resp.ParserID != chunkMethod || resp.IngestionStatus != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {
- t.Fatalf("response = %+v, want method=%s ingestion_status=UNSTART counts=0", resp, chunkMethod)
- }
-
- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
- if doc.ParserID != chunkMethod {
- t.Fatalf("parser_id = %q, want %q", doc.ParserID, chunkMethod)
- }
- if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.Progress != 0 {
- t.Fatalf("doc counters/progress = token:%d chunk:%d progress:%f, want zero", doc.TokenNum, doc.ChunkNum, doc.Progress)
- }
- kb, _ := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
- if kb.TokenNum != 0 || kb.ChunkNum != 0 {
- t.Fatalf("kb counters = token:%d chunk:%d, want zero", kb.TokenNum, kb.ChunkNum)
- }
-}
-
-func TestResetDocumentForReparseSkipsSecondCounterDecrement(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
- insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
-
- ctx := t.Context()
- staleDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
- if err != nil {
- t.Fatalf("get doc: %v", err)
- }
-
- svc := testDocumentService(t)
- parserID := "manual"
- if err = svc.resetDocumentForReparse(ctx, staleDoc, "tenant-1", &parserID, nil); err != nil {
- t.Fatalf("first resetDocumentForReparse failed: %v", err)
- }
- if err = svc.resetDocumentForReparse(ctx, staleDoc, "tenant-1", &parserID, nil); err != nil {
- t.Fatalf("second resetDocumentForReparse failed: %v", err)
- }
+func TestUpdateDatasetDocumentParserConfigPreservesParseResults(t *testing.T) {
+ builtin, pipeline := 1, 2
+ pipelineID := "1234567890abcdef1234567890abcdef"
+ cases := []struct {
+ name string
+ request UpdateDatasetDocumentRequest
+ present map[string]bool
+ wantParser string
+ wantPipeline string
+ }{
+ {name: "config only", wantParser: "naive"},
+ {name: "same builtin", request: UpdateDatasetDocumentRequest{ParseType: &builtin, ParserID: sptr("naive")}, present: map[string]bool{"parse_type": true, "parser_id": true}, wantParser: "naive"},
+ {name: "different builtin", request: UpdateDatasetDocumentRequest{ParseType: &builtin, ParserID: sptr("manual")}, present: map[string]bool{"parse_type": true, "parser_id": true}, wantParser: "manual"},
+ {name: "pipeline", request: UpdateDatasetDocumentRequest{ParseType: &pipeline, PipelineID: &pipelineID}, present: map[string]bool{"parse_type": true, "pipeline_id": true}, wantParser: "naive", wantPipeline: pipelineID},
+ {name: "chunk method", request: UpdateDatasetDocumentRequest{ChunkMethod: sptr("manual")}, present: map[string]bool{"chunk_method": true}, wantParser: "manual"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
+ insertTestCanvas(t, pipelineID, "user-1")
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_duration", 42).Error; err != nil {
+ t.Fatalf("prepare document: %v", err)
+ }
- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
- if doc.TokenNum != 0 || doc.ChunkNum != 0 {
- t.Fatalf("doc counters = token:%d chunk:%d, want zero", doc.TokenNum, doc.ChunkNum)
- }
- kb, _ := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
- if kb.TokenNum != 0 || kb.ChunkNum != 0 {
- t.Fatalf("kb counters = token:%d chunk:%d, want zero after duplicate reset", kb.TokenNum, kb.ChunkNum)
+ request := tc.request
+ request.ParserConfig = map[string]any{
+ "Extractor:AutoExtractDefault": map[string]any{"keywords": map[string]any{"top_n": 5}},
+ }
+ present := tc.present
+ if present == nil {
+ present = map[string]bool{}
+ }
+ present["parser_config"] = true
+ engine := &rerunDeleteDocEngine{}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+ resp, code, err := svc.UpdateDatasetDocument(t.Context(), "user-1", "kb-1", "doc-1", &request, present)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)
+ }
+ if resp.ParserID != tc.wantParser || resp.IngestionStatus != string(common.COMPLETED) || resp.TokenCount != 10 || resp.ChunkCount != 5 || resp.Progress != 0.75 || resp.ProcessDuration != 42 {
+ t.Fatalf("response changed parse results: %+v", resp)
+ }
+ if tc.wantPipeline != "" && (resp.PipelineID == nil || *resp.PipelineID != tc.wantPipeline) {
+ t.Fatalf("pipeline_id = %v, want %q", resp.PipelineID, tc.wantPipeline)
+ }
+ doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("get document: %v", err)
+ }
+ if doc.TokenNum != 10 || doc.ChunkNum != 5 || doc.Progress != 0.75 || doc.ProcessDuration != 42 {
+ t.Fatalf("persisted parse results changed: %+v", doc)
+ }
+ extractor, ok := doc.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("extractor config not saved: %#v", doc.ParserConfig)
+ }
+ keywords, ok := extractor["keywords"].(map[string]interface{})
+ if !ok || keywords["top_n"] != float64(5) {
+ t.Fatalf("parser config not saved: %#v", doc.ParserConfig)
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatalf("get dataset: %v", err)
+ }
+ if kb.TokenNum != 10 || kb.ChunkNum != 5 {
+ t.Fatalf("dataset counters changed: token=%d chunk=%d", kb.TokenNum, kb.ChunkNum)
+ }
+ if engine.deleteCalls != 0 || engine.search != nil {
+ t.Fatalf("existing chunks accessed during settings update: deletes=%d search=%+v", engine.deleteCalls, engine.search)
+ }
+ task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil || task == nil || task.ID != "task-1" || task.Status != common.COMPLETED {
+ t.Fatalf("existing ingestion task changed: task=%+v err=%v", task, err)
+ }
+ })
}
}
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 6a7d219c46..93dbc41491 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -491,6 +491,8 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
applyAutoMetadataSettings: 'Apply global auto-metadata settings',
parseFileTip: 'Are you sure to parse?',
clearChunksReparseTip: 'Are you sure to clear chunks to re-parse?',
+ parserConfigReparseTip:
+ 'Existing chunks were generated using the previous parser settings. Would you like to clear them and re-parse this document using the saved settings?',
parseFile: 'Parse file',
emptyMetadata: 'No metadata',
metadataField: 'Metadata field',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 2085d7d29e..4dbdeaae6c 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -438,6 +438,8 @@ export default {
applyAutoMetadataSettings: '应用全局自动元数据设置',
parseFileTip: '您确定要解析吗?',
clearChunksReparseTip: '您确定要清空分块重新解析吗?',
+ parserConfigReparseTip:
+ '现有分块由之前的解析设置生成。是否清空现有分块,并使用已保存的设置重新解析此文档?',
parseFile: '解析文件',
emptyMetadata: '无元数据',
localUpload: '本地上传',
diff --git a/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx b/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx
index e96ba58e02..d5767e42d0 100644
--- a/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx
+++ b/web/src/pages/dataset/dataset/change-parser-dialog.test.tsx
@@ -3,6 +3,7 @@ import { ChangeParserDialog } from './change-parser-dialog';
import { useChangeDocumentParser } from './use-change-document-parser';
const mockSave = jest.fn().mockResolvedValue(0);
+const mockRun = jest.fn().mockResolvedValue(0);
const mockDsl = {
graph: {
nodes: [
@@ -40,6 +41,7 @@ jest.mock('@/hooks/use-agent-request', () => ({
}),
}));
jest.mock('@/hooks/use-document-request', () => ({
+ useRunDocument: () => ({ runDocumentByIds: mockRun, loading: false }),
useSetDocumentPipelineParser: () => ({
setDocumentPipelineParser: mockSave,
loading: false,
@@ -93,12 +95,19 @@ jest.mock('react-i18next', () => ({
useTranslation: () => ({ t: (key: string) => key }),
}));
-function Harness({ builtin = false }: { builtin?: boolean }) {
+function Harness({
+ builtin = false,
+ chunkCount = 0,
+}: {
+ builtin?: boolean;
+ chunkCount?: number;
+}) {
const state = useChangeDocumentParser();
const openDialog = () => {
state.showChangeParserModal({
id: 'document',
dataset_id: 'dataset',
+ chunk_count: chunkCount,
chunk_method: builtin ? 'general' : '',
pipeline_id: builtin ? '' : 'pipeline',
parser_config: builtin
@@ -128,10 +137,60 @@ function Harness({ builtin = false }: { builtin?: boolean }) {
}
beforeEach(() => {
- mockSave.mockClear();
+ mockSave.mockReset().mockResolvedValue(0);
+ mockRun.mockReset().mockResolvedValue(0);
mockCustomDsl = mockDsl;
});
+it('keeps existing chunks when re-parsing is declined after saving', async () => {
+ render(<Harness builtin chunkCount={5} />);
+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));
+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));
+ expect(
+ await screen.findByText('knowledgeDetails.parserConfigReparseTip'),
+ ).toBeInTheDocument();
+ expect(mockSave).toHaveBeenCalledTimes(1);
+ expect(mockRun).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByRole('button', { name: 'common.no' }));
+ await waitFor(() =>
+ expect(
+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),
+ ).not.toBeInTheDocument(),
+ );
+ expect(mockRun).not.toHaveBeenCalled();
+});
+
+it('re-parses with the saved document settings only after confirmation', async () => {
+ render(<Harness chunkCount={5} />);
+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));
+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));
+ const confirm = await screen.findByRole('button', { name: 'common.yes' });
+ expect(mockSave).toHaveBeenCalledTimes(1);
+ expect(mockRun).not.toHaveBeenCalled();
+ fireEvent.click(confirm);
+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));
+ expect(mockRun).toHaveBeenCalledWith({
+ documentIds: ['document'],
+ run: 1,
+ option: { delete: true, apply_kb: false },
+ });
+});
+
+it('does not offer re-parsing when saving fails', async () => {
+ mockSave.mockResolvedValueOnce(102);
+ render(<Harness builtin chunkCount={5} />);
+ fireEvent.click(screen.getByRole('button', { name: 'Open' }));
+ fireEvent.click(screen.getByRole('button', { name: 'common.save' }));
+ await waitFor(() => expect(mockSave).toHaveBeenCalledTimes(1));
+ expect(
+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),
+ ).not.toBeInTheDocument();
+ expect(
+ screen.getByRole('button', { name: 'common.save' }),
+ ).toBeInTheDocument();
+ expect(mockRun).not.toHaveBeenCalled();
+});
+
it('saves the built-in parser', async () => {
render(<Harness builtin />);
fireEvent.click(screen.getByRole('button', { name: 'Open' }));
@@ -144,6 +203,10 @@ it('saves the built-in parser', async () => {
parse_type: 1,
}),
);
+ expect(
+ screen.queryByText('knowledgeDetails.parserConfigReparseTip'),
+ ).not.toBeInTheDocument();
+ expect(mockRun).not.toHaveBeenCalled();
});
it('saves after switching from a built-in parser to a pipeline', async () => {
diff --git a/web/src/pages/dataset/dataset/use-change-document-parser.ts b/web/src/pages/dataset/dataset/use-change-document-parser.ts
index 8d6a5b1ce8..267a43e475 100644
--- a/web/src/pages/dataset/dataset/use-change-document-parser.ts
+++ b/web/src/pages/dataset/dataset/use-change-document-parser.ts
@@ -1,11 +1,18 @@
+import { Modal } from '@/components/ui/modal/modal';
import { useSetModalState } from '@/hooks/common-hooks';
-import { useSetDocumentPipelineParser } from '@/hooks/use-document-request';
+import {
+ useRunDocument,
+ useSetDocumentPipelineParser,
+} from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
import { IChangeParserRequestBody } from '@/interfaces/request/document';
import { useCallback, useState } from 'react';
+import { useTranslation } from 'react-i18next';
export const useChangeDocumentParser = () => {
+ const { t } = useTranslation();
const { setDocumentPipelineParser, loading } = useSetDocumentPipelineParser();
+ const { runDocumentByIds } = useRunDocument();
const [record, setRecord] = useState<IDocumentInfo>({} as IDocumentInfo);
const {
@@ -29,14 +36,36 @@ export const useChangeDocumentParser = () => {
});
if (ret === 0) {
hideChangeParserModal();
+ if (record.chunk_count > 0) {
+ Modal.confirm({
+ title: t('knowledgeDetails.parseFile'),
+ content: t('knowledgeDetails.parserConfigReparseTip'),
+ okText: t('common.yes'),
+ cancelText: t('common.no'),
+ onOk: async () => {
+ try {
+ await runDocumentByIds({
+ documentIds: [record.id],
+ run: 1,
+ option: { delete: true, apply_kb: false },
+ });
+ } catch (error) {
+ console.warn(error);
+ }
+ },
+ });
+ }
}
}
},
[
record?.id,
record?.dataset_id,
+ record?.chunk_count,
+ runDocumentByIds,
setDocumentPipelineParser,
hideChangeParserModal,
+ t,
],
);