Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: record generated wiki page versions (#19636)

buua4361cb339ced763
diff --git a/internal/entity/file_commit.go b/internal/entity/file_commit.go
index 585ce50ab7..1f262180a7 100644
--- a/internal/entity/file_commit.go
+++ b/internal/entity/file_commit.go
@@ -130,3 +130,22 @@ type WikiPageCommit struct {
CreateDate *time.Time `json:"create_date,omitempty"`
UserNickname string `json:"user_nickname"`
}
+
+// WikiPageCommitDetail is the artifact-page commit detail returned by the
+// dataset commit endpoint. It mirrors Python's flat response shape consumed
+// by the wiki version-history UI.
+type WikiPageCommitDetail struct {
+ ID string `json:"id"`
+ TenantID string `json:"tenant_id"`
+ KBID string `json:"kb_id"`
+ PageType string `json:"page_type_kwd"`
+ Slug string `json:"slug"`
+ UserID *string `json:"user_id"`
+ UserNickname string `json:"user_nickname"`
+ Title string `json:"title"`
+ Comments string `json:"comments"`
+ Diff string `json:"diff"`
+ ContentAfter string `json:"content_after"`
+ CreateTime *int64 `json:"create_time,omitempty"`
+ CreateDate *time.Time `json:"create_date,omitempty"`
+}
diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go
index 125a50bb80..9afaf8ea8b 100644
--- a/internal/handler/dataset_artifact.go
+++ b/internal/handler/dataset_artifact.go
@@ -184,7 +184,6 @@ func (h *DatasetArtifactHandler) UpdateArtifact(c *gin.Context) {
}
if _, cerr := h.fileCommitSvc.RecordPageEdit(c.Request.Context(), file.PageEditCommitInput{
DatasetID: datasetID,
- DocID: pageType + "/" + slug,
Slug: slug,
PageType: pageType,
Title: title,
diff --git a/internal/handler/file_commit.go b/internal/handler/file_commit.go
index 9249b4a62e..f7f26c4a00 100644
--- a/internal/handler/file_commit.go
+++ b/internal/handler/file_commit.go
@@ -34,6 +34,7 @@ type fileCommitService interface {
CreateCommit(ctx context.Context, folderID, authorID, message string, changes []entity.FileChange) (*entity.FileCommit, error)
ListCommits(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error)
GetCommit(ctx context.Context, commitID string) (*entity.FileCommit, error)
+ GetPageCommitDetail(ctx context.Context, datasetID, commitID string) (*entity.WikiPageCommitDetail, error)
ListCommitFiles(ctx context.Context, commitID string) ([]*entity.FileCommitItem, error)
DiffCommits(ctx context.Context, fromID, toID string) ([]entity.DiffEntry, error)
GetUncommittedChanges(ctx context.Context, folderID string) ([]entity.DiffEntry, error)
@@ -359,10 +360,23 @@ func (h *FileCommitHandler) GetCommit(c *gin.Context) {
return
}
- if commit.FolderID != folderID {
+ // Workspace commits store the workspace folder id. Wiki/skill page audit
+ // commits are scoped by dataset instead, so the dataset route must accept
+ // the dataset id as the commit scope after the resolver has authorized it.
+ datasetID := c.Param("dataset_id")
+ if commit.FolderID != folderID && (datasetID == "" || commit.FolderID != datasetID) {
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Commit not found in workspace")
return
}
+ if datasetID != "" && commit.Title != nil {
+ detail, err := h.commitService.GetPageCommitDetail(ctx, datasetID, commitID)
+ if err != nil {
+ common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Commit not found")
+ return
+ }
+ common.SuccessWithData(c, detail, common.CodeSuccess.Message())
+ return
+ }
items, err := h.commitService.ListCommitFiles(ctx, commitID)
if err != nil {
diff --git a/internal/handler/file_commit_test.go b/internal/handler/file_commit_test.go
index 6de965c252..83d3226781 100644
--- a/internal/handler/file_commit_test.go
+++ b/internal/handler/file_commit_test.go
@@ -38,6 +38,7 @@ type mockFileCommitSvc struct {
createCommitFn func(ctx context.Context, folderID, authorID, message string, changes []entity.FileChange) (*entity.FileCommit, error)
listCommitsFn func(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error)
getCommitFn func(ctx context.Context, commitID string) (*entity.FileCommit, error)
+ getPageCommitDetailFn func(ctx context.Context, datasetID, commitID string) (*entity.WikiPageCommitDetail, error)
listCommitFilesFn func(ctx context.Context, commitID string) ([]*entity.FileCommitItem, error)
diffCommitsFn func(ctx context.Context, fromID, toID string) ([]entity.DiffEntry, error)
getUncommittedChangesFn func(ctx context.Context, folderID string) ([]entity.DiffEntry, error)
@@ -78,6 +79,13 @@ func (m *mockFileCommitSvc) GetCommit(ctx context.Context, commitID string) (*en
return &entity.FileCommit{ID: commitID, FolderID: "folder-1", Message: "test commit", AuthorID: "u1", FileCount: 1}, nil
}
+func (m *mockFileCommitSvc) GetPageCommitDetail(ctx context.Context, datasetID, commitID string) (*entity.WikiPageCommitDetail, error) {
+ if m.getPageCommitDetailFn != nil {
+ return m.getPageCommitDetailFn(ctx, datasetID, commitID)
+ }
+ return &entity.WikiPageCommitDetail{ID: commitID, KBID: datasetID}, nil
+}
+
func (m *mockFileCommitSvc) ListCommitFiles(ctx context.Context, commitID string) ([]*entity.FileCommitItem, error) {
if m.listCommitFilesFn != nil {
return m.listCommitFilesFn(ctx, commitID)
@@ -197,6 +205,10 @@ func fileCommitRouter(h *FileCommitHandler, userID string) *gin.Engine {
c.Params = append(c.Params, gin.Param{Key: "folder_id", Value: c.Param("dataset_id")})
c.Next()
}, h.ListCommits)
+ r.GET("/api/v1/datasets/:dataset_id/commits/:commit_id", func(c *gin.Context) {
+ c.Params = append(c.Params, gin.Param{Key: "folder_id", Value: "folder-1"})
+ c.Next()
+ }, h.GetCommit)
return r
}
@@ -488,6 +500,52 @@ func TestFileCommit_GetCommit_NotFound(t *testing.T) {
}
}
+func TestFileCommit_GetDatasetPageCommit_Success(t *testing.T) {
+ r, mock := setupFileCommitTest("user-1")
+ title := "page edit"
+ mock.getCommitFn = func(ctx context.Context, commitID string) (*entity.FileCommit, error) {
+ return &entity.FileCommit{
+ ID: commitID,
+ FolderID: "kb-1", // Page-edit commits are scoped by dataset id.
+ Message: title,
+ Title: &title,
+ }, nil
+ }
+ mock.getPageCommitDetailFn = func(ctx context.Context, datasetID, commitID string) (*entity.WikiPageCommitDetail, error) {
+ if datasetID != "kb-1" || commitID != "commit-1" {
+ t.Fatalf("unexpected page commit lookup: dataset=%s commit=%s", datasetID, commitID)
+ }
+ return &entity.WikiPageCommitDetail{
+ ID: commitID,
+ KBID: datasetID,
+ Title: title,
+ Comments: "save note",
+ Diff: "-old\n+new",
+ ContentAfter: "new",
+ }, nil
+ }
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/api/v1/datasets/kb-1/commits/commit-1", nil)
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ if code := decodeCode(t, w); code != float64(common.CodeSuccess) {
+ t.Fatalf("expected code %d, got %v", common.CodeSuccess, code)
+ }
+ var resp struct {
+ Data entity.WikiPageCommitDetail `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if resp.Data.Title != title || resp.Data.Diff != "-old\n+new" || resp.Data.ContentAfter != "new" {
+ t.Fatalf("unexpected page commit detail: %+v", resp.Data)
+ }
+}
+
func TestFileCommit_ListCommitFiles_Success(t *testing.T) {
r, _ := setupFileCommitTest("user-1")
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 31df1251d5..5ae0839f2d 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/file"
"ragflow/internal/service/nav"
"go.uber.org/zap"
@@ -83,7 +84,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
c := &Consumer{
scheduler: scheduler,
reader: engineReader{eng: engine.Get()},
- writer: engineWriter{eng: engine.Get()},
+ writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService()},
factory: defaultDeduperFactory,
contributions: newWikiContributionStore(engine.Get()),
ttl: 2 * time.Minute,
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 818a6a3f93..de2154d7c1 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -31,7 +31,9 @@ import (
"ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
+ "ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/file"
"ragflow/internal/utility"
)
@@ -130,7 +132,12 @@ type StructureBucket struct {
// obtained via engine.Get(); the storage schema lives behind the engine
// abstraction rather than in this package.
type engineWriter struct {
- eng engine.DocEngine
+ eng engine.DocEngine
+ commitService pageCommitter
+}
+
+type pageCommitter interface {
+ RecordPageEdit(context.Context, file.PageEditCommitInput) (*entity.FileCommit, error)
}
func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb string, ids []string) error {
@@ -186,6 +193,20 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
if len(products) == 0 {
return nil
}
+ var existingWikiPageContent map[string]string
+ canRecordWikiCommits := w.commitService != nil
+ if canRecordWikiCommits {
+ var err error
+ existingWikiPageContent, err = w.loadWikiPageContent(ctx, tenant, kb, products)
+ if err != nil {
+ // The compiled page write must not fail because the audit lookup is
+ // unavailable. Skip auditing this batch rather than recording an
+ // incorrect add commit for a page whose old content is unknown.
+ common.Warn("knowledge_compile: failed to load Wiki page content for version history",
+ zap.String("kb_id", kb), zap.Error(err))
+ canRecordWikiCommits = false
+ }
+ }
// Dataset-level telemetry: break the merged set down by compile_kwd so a
// missing wiki_page at query time can be traced to "WriteMerged never
// received any wiki_page products" (generation/merge bug) rather than
@@ -264,7 +285,178 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
return err
})
}
- return runCompilerJobs(ctx, jobs)
+ if err := runCompilerJobs(ctx, jobs); err != nil {
+ return err
+ }
+ if canRecordWikiCommits {
+ w.recordWikiPageCommits(ctx, kb, products, existingWikiPageContent)
+ }
+ return nil
+}
+
+type wikiPageCommitTarget struct {
+ key string
+ fullSlug string
+ bareSlug string
+ pageType string
+}
+
+// loadWikiPageContent returns the currently persisted content for the page
+// products in products. A missing key means that this is the page's first
+// generated version, while an empty value means an existing page currently has
+// empty content; callers must preserve that distinction.
+func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string, products []kccommon.Product) (map[string]string, error) {
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ result := make(map[string]string)
+ if eng == nil {
+ return result, nil
+ }
+
+ pageSlugs := make(map[string]struct{})
+ for _, product := range products {
+ target, ok := wikiPageCommitTargetForProduct(product)
+ if ok {
+ pageSlugs[target.fullSlug] = struct{}{}
+ }
+ }
+ if len(pageSlugs) == 0 {
+ return result, nil
+ }
+
+ slugs := make([]string, 0, len(pageSlugs))
+ for slug := range pageSlugs {
+ slugs = append(slugs, slug)
+ }
+ sort.Strings(slugs)
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ const pageBatchSize = 2000
+ for offset := 0; ; offset += pageBatchSize {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{kb},
+ Offset: offset,
+ Limit: pageBatchSize,
+ SelectFields: []string{
+ "slug_kwd", "page_type_kwd", "md_with_weight", "content_with_weight",
+ },
+ Filter: map[string]interface{}{
+ "compile_kwd": compileKwdWikiPage,
+ "available_int": 1,
+ "kb_id": kb,
+ "slug_kwd": slugs,
+ },
+ OrderBy: (&types.OrderByExpr{}).Asc("slug_kwd"),
+ })
+ if err != nil {
+ return nil, err
+ }
+ if res == nil || len(res.Chunks) == 0 {
+ break
+ }
+ for _, row := range res.Chunks {
+ target, ok := wikiPageCommitTargetForRow(
+ pageEngineString(row["page_type_kwd"]),
+ pageEngineString(row["slug_kwd"]),
+ )
+ if !ok {
+ continue
+ }
+ content := pageEngineString(row["md_with_weight"])
+ if content == "" {
+ content = pageEngineString(row["content_with_weight"])
+ }
+ result[target.key] = content
+ }
+ if len(res.Chunks) < pageBatchSize {
+ break
+ }
+ }
+ return result, nil
+}
+
+// recordWikiPageCommits records generated page revisions after their merged
+// rows have been written. Sections and other Wiki products are intentionally
+// excluded because Python's generated-page history is page-level only.
+func (w engineWriter) recordWikiPageCommits(ctx context.Context, kb string, products []kccommon.Product, existing map[string]string) {
+ if w.commitService == nil {
+ return
+ }
+ seen := make(map[string]struct{})
+ for _, product := range products {
+ target, ok := wikiPageCommitTargetForProduct(product)
+ if !ok {
+ continue
+ }
+ if _, duplicate := seen[target.key]; duplicate {
+ continue
+ }
+ seen[target.key] = struct{}{}
+
+ _, err := w.commitService.RecordPageEdit(ctx, file.PageEditCommitInput{
+ DatasetID: kb,
+ Slug: target.bareSlug,
+ PageType: target.pageType,
+ Title: "Regenerated by artifact compilation",
+ Comments: "Auto-update via incremental wiki compilation",
+ OldContent: existing[target.key],
+ NewContent: product.Content,
+ })
+ if err != nil {
+ common.Warn("knowledge_compile: failed to record generated Wiki page version",
+ zap.String("kb_id", kb),
+ zap.String("slug", target.fullSlug),
+ zap.Error(err))
+ }
+ }
+}
+
+func wikiPageCommitTargetForProduct(product kccommon.Product) (wikiPageCommitTarget, bool) {
+ if product.Variant != kccommon.VariantWiki || strings.ToLower(strings.TrimSpace(metaString(product.Meta, "kind"))) != "page" {
+ return wikiPageCommitTarget{}, false
+ }
+ return wikiPageCommitTargetForRow(
+ metaString(product.Meta, "page_type"),
+ metaString(product.Meta, "slug"),
+ )
+}
+
+func wikiPageCommitTargetForRow(pageType, slug string) (wikiPageCommitTarget, bool) {
+ pageType = strings.TrimSpace(pageType)
+ slug = strings.TrimSpace(slug)
+ if pageType == "" || slug == "" {
+ return wikiPageCommitTarget{}, false
+ }
+ bareSlug := strings.TrimPrefix(slug, pageType+"/")
+ if bareSlug == "" {
+ return wikiPageCommitTarget{}, false
+ }
+ return wikiPageCommitTarget{
+ key: pageType + "\x00" + bareSlug,
+ fullSlug: pageType + "/" + bareSlug,
+ bareSlug: bareSlug,
+ pageType: pageType,
+ }, true
+}
+
+func pageEngineString(value any) string {
+ switch typed := value.(type) {
+ case string:
+ return typed
+ case []string:
+ if len(typed) > 0 {
+ return typed[0]
+ }
+ case []any:
+ if len(typed) > 0 {
+ if text, ok := typed[0].(string); ok {
+ return text
+ }
+ }
+ }
+ return ""
}
// WriteMergedStructure writes the dataset-level structure merged rows for a KB
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 1c6a4e4ba3..861def5426 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -5,9 +5,59 @@ import (
"testing"
"time"
+ "ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/file"
)
+type recordingPageCommitter struct {
+ inputs []file.PageEditCommitInput
+}
+
+func (c *recordingPageCommitter) RecordPageEdit(_ context.Context, input file.PageEditCommitInput) (*entity.FileCommit, error) {
+ c.inputs = append(c.inputs, input)
+ return &entity.FileCommit{ID: "commit-1"}, nil
+}
+
+func TestWriteMergedRecordsGeneratedPageCommitOnly(t *testing.T) {
+ committer := &recordingPageCommitter{}
+ w := engineWriter{eng: &fakeEngine{}, commitService: committer}
+ products := []kccommon.Product{
+ {
+ Variant: kccommon.VariantWiki,
+ Content: "# Alpha\n\nBody",
+ Meta: map[string]any{
+ "kind": "page",
+ "slug": "entity/alpha",
+ "page_type": "entity",
+ },
+ },
+ {
+ Variant: kccommon.VariantWiki,
+ Content: "Body",
+ Meta: map[string]any{
+ "kind": "section",
+ "slug": "body",
+ "page_type": "entity",
+ },
+ },
+ }
+
+ if err := w.WriteMerged(t.Context(), "tenant-1", "kb-1", products); err != nil {
+ t.Fatalf("WriteMerged: %v", err)
+ }
+ if len(committer.inputs) != 1 {
+ t.Fatalf("generated page commits = %d, want 1", len(committer.inputs))
+ }
+ input := committer.inputs[0]
+ if input.DatasetID != "kb-1" || input.PageType != "entity" || input.Slug != "alpha" {
+ t.Fatalf("unexpected generated page identity: %+v", input)
+ }
+ if input.OldContent != "" || input.NewContent != "# Alpha\n\nBody" {
+ t.Fatalf("unexpected generated page content: %+v", input)
+ }
+}
+
// TestMergedChunkMapKeepsWikiFields locks the fix for the merged-row metadata
// gap: the dataset-level merged row written by mergedChunkMap must carry the
// wiki page fields (page_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the
diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.go
index 180cb70958..2c266575ec 100644
--- a/internal/service/file/file_commit.go
+++ b/internal/service/file/file_commit.go
@@ -24,6 +24,8 @@ import (
"fmt"
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
+ enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/storage"
"ragflow/internal/utility"
@@ -42,6 +44,7 @@ type FileCommitService struct {
commitDAO *dao.FileCommitDAO
commitItemDAO *dao.FileCommitItemDAO
fileDAO *dao.FileDAO
+ kbDAO *dao.KnowledgebaseDAO
}
// NewFileCommitService create file commit service
@@ -50,6 +53,7 @@ func NewFileCommitService() *FileCommitService {
commitDAO: dao.NewFileCommitDAO(),
commitItemDAO: dao.NewFileCommitItemDAO(),
fileDAO: dao.NewFileDAO(),
+ kbDAO: dao.NewKnowledgebaseDAO(),
}
}
@@ -240,7 +244,6 @@ func (s *FileCommitService) CreateCommit(ctx context.Context, folderID, authorID
// page edit as an audit commit.
type PageEditCommitInput struct {
DatasetID string // knowledgebase scope, stored on the commit for isolation
- DocID string // ES doc id of the page content
Slug string
PageType string
Title string
@@ -259,10 +262,15 @@ func wikiFileID(datasetID, pageType, slug string) string {
var pageCommitSeq atomic.Uint64
+const (
+ wikiContentStorage = "minio"
+ wikiCommitBucketPrefix = ".wiki_commits"
+)
+
// RecordPageEdit records a wiki/skill page edit as an audit commit with a
// git-style parent chain (each edit points at the previous commit for the same
-// page). The new content_after is referenced in ES by doc_id; a unified diff of
-// old vs new content is stored on the commit item. The commit is scoped to the
+// page). The post-save content is stored as a content-addressed object and a
+// unified diff is stored on the commit item. The commit is scoped to the
// dataset via FolderID and a derived page file key so page histories never cross
// knowledgebase boundaries.
//
@@ -275,21 +283,31 @@ func (s *FileCommitService) RecordPageEdit(ctx context.Context, in PageEditCommi
commitID := utility.GenerateUUID()
diffText := unifiedDiff(in.OldContent, in.NewContent)
- contentAfterStorage := "es"
- contentAfterLocation := in.DocID
+ if diffText == "" {
+ return nil, nil
+ }
+ contentAfterStorage, contentAfterLocation := s.storePageContent(ctx, in.DatasetID, in.NewContent)
slugKwd := in.Slug
pageTypeKwd := in.PageType
+ operation := "modify"
+ if in.OldContent == "" {
+ operation = "add"
+ }
item := &entity.FileCommitItem{
- ID: utility.GenerateUUID(),
- CommitID: commitID,
- FileID: fileID,
- Operation: "modify",
- Diff: &diffText,
- ContentAfterStorage: &contentAfterStorage,
- ContentAfterLocation: &contentAfterLocation,
- SlugKwd: &slugKwd,
- PageTypeKwd: &pageTypeKwd,
+ ID: utility.GenerateUUID(),
+ CommitID: commitID,
+ FileID: fileID,
+ Operation: operation,
+ Diff: &diffText,
+ SlugKwd: &slugKwd,
+ PageTypeKwd: &pageTypeKwd,
+ }
+ if contentAfterStorage != "" {
+ item.ContentAfterStorage = &contentAfterStorage
+ }
+ if contentAfterLocation != "" {
+ item.ContentAfterLocation = &contentAfterLocation
}
var commit *entity.FileCommit
@@ -338,6 +356,23 @@ func (s *FileCommitService) RecordPageEdit(ctx context.Context, in PageEditCommi
return commit, nil
}
+// storePageContent saves a page version in the same dataset-scoped object
+// layout used by Python's MinIO-backed artifact history. Missing storage is
+// tolerated because page updates and audit rows are intentionally best-effort.
+func (s *FileCommitService) storePageContent(ctx context.Context, datasetID, content string) (string, string) {
+ storageImpl := storage.GetStorageFactory().GetStorage()
+ if storageImpl == nil {
+ return "", ""
+ }
+ hash := sha256.Sum256([]byte(content))
+ location := wikiCommitBucketPrefix + "/" + hex.EncodeToString(hash[:])
+ if err := storageImpl.Put(ctx, datasetID, location, []byte(content)); err != nil {
+ common.Warn("failed to store wiki commit content", zap.Error(err))
+ return "", ""
+ }
+ return wikiContentStorage, location
+}
+
// pageCommitLocks is a per-page-file-key mutex registry that serializes
// RecordPageEdit calls for the same page. Entries are retained for the process
// lifetime (bounded by the number of distinct pages edited).
@@ -411,18 +446,20 @@ func (s *FileCommitService) ListPageCommits(ctx context.Context, datasetID, page
return rows, total, nil
}
-// unifiedDiff produces a simple line-based unified diff between two texts.
+type unifiedDiffLine struct {
+ prefix byte
+ text string
+ oldLine int
+ newLine int
+}
+
+// unifiedDiff produces a standard unified diff that can be parsed by the
+// frontend's wiki diff renderer and Python's artifact history implementation.
func unifiedDiff(oldText, newText string) string {
oldLines := strings.Split(oldText, "\n")
newLines := strings.Split(newText, "\n")
- const maxCtx = 3
- type hunkLine struct {
- prefix string
- text string
- }
- var hunks []hunkLine
-
+ const contextLines = 3
// Longest common subsequence over lines, then render the diff.
cur := make([][]int, len(oldLines)+1)
for i := range cur {
@@ -440,53 +477,102 @@ func unifiedDiff(oldText, newText string) string {
}
}
+ lines := make([]unifiedDiffLine, 0, len(oldLines)+len(newLines))
i, j := 0, 0
for i < len(oldLines) && j < len(newLines) {
if oldLines[i] == newLines[j] {
+ lines = append(lines, unifiedDiffLine{' ', oldLines[i], i + 1, j + 1})
i++
j++
} else if cur[i+1][j] >= cur[i][j+1] {
- hunks = append(hunks, hunkLine{"-", oldLines[i]})
+ lines = append(lines, unifiedDiffLine{'-', oldLines[i], i + 1, 0})
i++
} else {
- hunks = append(hunks, hunkLine{"+", newLines[j]})
+ lines = append(lines, unifiedDiffLine{'+', newLines[j], 0, j + 1})
j++
}
}
for ; i < len(oldLines); i++ {
- hunks = append(hunks, hunkLine{"-", oldLines[i]})
+ lines = append(lines, unifiedDiffLine{'-', oldLines[i], i + 1, 0})
}
for ; j < len(newLines); j++ {
- hunks = append(hunks, hunkLine{"+", newLines[j]})
+ lines = append(lines, unifiedDiffLine{'+', newLines[j], 0, j + 1})
}
- if len(hunks) == 0 {
+ changed := make([]int, 0)
+ for index, line := range lines {
+ if line.prefix != ' ' {
+ changed = append(changed, index)
+ }
+ }
+ if len(changed) == 0 {
return ""
}
- if len(hunks) > maxCtx*2 {
- var b strings.Builder
- for k := 0; k < maxCtx; k++ {
- b.WriteString(hunks[k].prefix)
- b.WriteString(hunks[k].text)
- b.WriteString("\n")
+
+ // Group nearby changes into hunks with up to three context lines, matching
+ // the format produced by Python's difflib.unified_diff.
+ type diffHunk struct {
+ start int
+ end int
+ }
+ hunks := make([]diffHunk, 0)
+ for _, index := range changed {
+ start := index - contextLines
+ if start < 0 {
+ start = 0
}
- b.WriteString("... (" + fmt.Sprintf("%d", len(hunks)-maxCtx*2) + " lines omitted) ...\n")
- for k := len(hunks) - maxCtx; k < len(hunks); k++ {
- b.WriteString(hunks[k].prefix)
- b.WriteString(hunks[k].text)
- b.WriteString("\n")
+ end := index + contextLines
+ if end >= len(lines) {
+ end = len(lines) - 1
+ }
+ if len(hunks) > 0 && start <= hunks[len(hunks)-1].end+1 {
+ if end > hunks[len(hunks)-1].end {
+ hunks[len(hunks)-1].end = end
+ }
+ continue
}
- return b.String()
+ hunks = append(hunks, diffHunk{start: start, end: end})
}
+
var b strings.Builder
- for _, h := range hunks {
- b.WriteString(h.prefix)
- b.WriteString(h.text)
- b.WriteString("\n")
+ b.WriteString("--- a\n+++ b\n")
+ for _, hunk := range hunks {
+ oldStart, oldCount := diffRange(lines, hunk.start, hunk.end, false, len(oldLines))
+ newStart, newCount := diffRange(lines, hunk.start, hunk.end, true, len(newLines))
+ b.WriteString(fmt.Sprintf("@@ -%d,%d +%d,%d @@\n", oldStart, oldCount, newStart, newCount))
+ for index := hunk.start; index <= hunk.end; index++ {
+ b.WriteByte(lines[index].prefix)
+ b.WriteString(lines[index].text)
+ b.WriteByte('\n')
+ }
}
return b.String()
}
+// diffRange returns the first line number and line count for one side of a
+// unified diff hunk. A zero count is valid for pure insertions or deletions.
+func diffRange(lines []unifiedDiffLine, start, end int, newSide bool, totalLines int) (int, int) {
+ first := 0
+ count := 0
+ for index := start; index <= end; index++ {
+ lineNumber := lines[index].oldLine
+ if newSide {
+ lineNumber = lines[index].newLine
+ }
+ if lineNumber == 0 {
+ continue
+ }
+ if first == 0 {
+ first = lineNumber
+ }
+ count++
+ }
+ if first == 0 {
+ first = totalLines
+ }
+ return first, count
+}
+
// ListCommits lists commits for a workspace folder with pagination
func (s *FileCommitService) ListCommits(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error) {
return s.commitDAO.ListByFolderID(ctx, dao.DB, folderID, page, pageSize, orderBy, desc)
@@ -497,6 +583,170 @@ func (s *FileCommitService) GetCommit(ctx context.Context, commitID string) (*en
return s.commitDAO.GetByID(ctx, dao.DB, commitID)
}
+// GetPageCommitDetail returns the flat artifact-page commit shape shared by
+// the Python API and the wiki version-history frontend.
+func (s *FileCommitService) GetPageCommitDetail(ctx context.Context, datasetID, commitID string) (*entity.WikiPageCommitDetail, error) {
+ commit, err := s.commitDAO.GetByID(ctx, dao.DB, commitID)
+ if err != nil {
+ return nil, err
+ }
+ if commit.FolderID != datasetID || commit.Title == nil {
+ return nil, common.ErrNotFound
+ }
+
+ items, err := s.commitItemDAO.ListByCommitID(ctx, dao.DB, commitID)
+ if err != nil {
+ return nil, err
+ }
+ if len(items) == 0 {
+ return nil, common.ErrNotFound
+ }
+ item := items[0]
+
+ tenantID := ""
+ if s.kbDAO != nil {
+ kb, kerr := s.kbDAO.GetByID(ctx, dao.DB, datasetID)
+ if kerr != nil {
+ return nil, kerr
+ }
+ tenantID = kb.TenantID
+ }
+
+ pageType := ""
+ if item.PageTypeKwd != nil {
+ pageType = *item.PageTypeKwd
+ }
+ slug := ""
+ if item.SlugKwd != nil {
+ slug = *item.SlugKwd
+ }
+ diff := ""
+ if item.Diff != nil {
+ diff = *item.Diff
+ }
+ title := ""
+ if commit.Title != nil {
+ title = *commit.Title
+ }
+ comments := ""
+ if commit.Comments != nil {
+ comments = *commit.Comments
+ }
+
+ var userID *string
+ if commit.AuthorID != "" {
+ authorID := commit.AuthorID
+ userID = &authorID
+ }
+ nickname := ""
+ if commit.AuthorID != "" {
+ if resolved, nerr := dao.NewUserDAO().GetNicknameByID(ctx, dao.DB, commit.AuthorID); nerr == nil {
+ nickname = resolved
+ }
+ }
+
+ return &entity.WikiPageCommitDetail{
+ ID: commit.ID,
+ TenantID: tenantID,
+ KBID: datasetID,
+ PageType: pageType,
+ Slug: slug,
+ UserID: userID,
+ UserNickname: nickname,
+ Title: title,
+ Comments: comments,
+ Diff: diff,
+ ContentAfter: s.readPageContent(ctx, datasetID, tenantID, pageType, slug, item),
+ CreateTime: commit.CreateTime,
+ CreateDate: commit.CreateDate,
+ }, nil
+}
+
+func (s *FileCommitService) readPageContent(ctx context.Context, datasetID, tenantID, pageType, slug string, item *entity.FileCommitItem) string {
+ if item != nil && item.ContentAfterStorage != nil && item.ContentAfterLocation != nil {
+ storageKind := *item.ContentAfterStorage
+ location := *item.ContentAfterLocation
+ switch storageKind {
+ case wikiContentStorage, "minio_storage":
+ if storageImpl := storage.GetStorageFactory().GetStorage(); storageImpl != nil {
+ if content, err := storageImpl.Get(ctx, datasetID, location); err == nil {
+ return string(content)
+ }
+ }
+ case "es":
+ if docEngine := engine.Get(); docEngine != nil && tenantID != "" {
+ raw, err := docEngine.GetChunk(ctx, wikiIndexName(tenantID), location, []string{datasetID})
+ if err == nil {
+ if content := pageContentValue(raw); content != "" {
+ return content
+ }
+ }
+ }
+ }
+ }
+
+ // Older Go page commits referenced the live page document instead of a
+ // content snapshot. Keep those records readable while all new commits use
+ // the snapshot path above.
+ return s.readCurrentPageContent(ctx, tenantID, datasetID, pageType, slug)
+}
+
+func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantID, datasetID, pageType, slug string) string {
+ if tenantID == "" || datasetID == "" || pageType == "" || slug == "" {
+ return ""
+ }
+ docEngine := engine.Get()
+ if docEngine == nil {
+ return ""
+ }
+ slugKwd := slug
+ if !strings.HasPrefix(slugKwd, pageType+"/") {
+ slugKwd = pageType + "/" + slugKwd
+ }
+ result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{wikiIndexName(tenantID)},
+ KbIDs: []string{datasetID},
+ Limit: 1,
+ SelectFields: []string{"md_with_weight", "content_with_weight"},
+ Filter: map[string]interface{}{
+ "compile_kwd": []string{"wiki_page"},
+ "page_type_kwd": []string{pageType},
+ "slug_kwd": []string{slugKwd},
+ "available_int": 1,
+ },
+ })
+ if err != nil || result == nil || len(result.Chunks) == 0 {
+ return ""
+ }
+ content := pageContentValue(result.Chunks[0]["md_with_weight"])
+ if content == "" {
+ content = pageContentValue(result.Chunks[0]["content_with_weight"])
+ }
+ return content
+}
+
+func pageContentValue(value interface{}) string {
+ switch v := value.(type) {
+ case string:
+ return v
+ case []string:
+ if len(v) > 0 {
+ return v[0]
+ }
+ case []interface{}:
+ if len(v) > 0 {
+ if first, ok := v[0].(string); ok {
+ return first
+ }
+ }
+ }
+ return ""
+}
+
+func wikiIndexName(tenantID string) string {
+ return "ragflow_" + tenantID
+}
+
// ListCommitFiles lists all file change items for a commit
func (s *FileCommitService) ListCommitFiles(ctx context.Context, commitID string) ([]*entity.FileCommitItem, error) {
return s.commitItemDAO.ListByCommitID(ctx, dao.DB, commitID)
diff --git a/internal/service/file/file_commit_page_test.go b/internal/service/file/file_commit_page_test.go
index 06bf37f760..d3d402b280 100644
--- a/internal/service/file/file_commit_page_test.go
+++ b/internal/service/file/file_commit_page_test.go
@@ -9,6 +9,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/storage"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
@@ -32,7 +33,12 @@ func newPageCommitTestDB(t *testing.T) *gorm.DB {
}
old := dao.DB
dao.DB = db
- t.Cleanup(func() { dao.DB = old })
+ oldStorage := storage.GetStorageFactory().GetStorage()
+ storage.GetStorageFactory().SetStorage(storage.NewMemoryStorage())
+ t.Cleanup(func() {
+ dao.DB = old
+ storage.GetStorageFactory().SetStorage(oldStorage)
+ })
return db
}
@@ -43,7 +49,6 @@ func TestRecordPageEdit_CreatesCommitAndItem(t *testing.T) {
in := PageEditCommitInput{
DatasetID: "kb1",
- DocID: "wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: "First edit",
@@ -91,11 +96,70 @@ func TestRecordPageEdit_CreatesCommitAndItem(t *testing.T) {
if it.Diff == nil || !strings.Contains(*it.Diff, "hello world, edited") {
t.Fatalf("expected diff containing new content, got %v", it.Diff)
}
- if it.ContentAfterStorage == nil || *it.ContentAfterStorage != "es" {
- t.Fatalf("expected content_after_storage es, got %v", it.ContentAfterStorage)
+ if it.ContentAfterStorage == nil || *it.ContentAfterStorage != "minio" {
+ t.Fatalf("expected content_after_storage minio, got %v", it.ContentAfterStorage)
}
- if it.ContentAfterLocation == nil || *it.ContentAfterLocation != "wiki/page-a" {
- t.Fatalf("expected content_after_location wiki/page-a, got %v", it.ContentAfterLocation)
+ if it.ContentAfterLocation == nil || !strings.HasPrefix(*it.ContentAfterLocation, ".wiki_commits/") {
+ t.Fatalf("expected content snapshot location, got %v", it.ContentAfterLocation)
+ }
+}
+
+func TestGetPageCommitDetailReadsContentSnapshot(t *testing.T) {
+ newPageCommitTestDB(t)
+ if err := dao.DB.AutoMigrate(&entity.Knowledgebase{}, &entity.User{}); err != nil {
+ t.Fatalf("migrate page detail models: %v", err)
+ }
+ status := string(entity.StatusValid)
+ if err := dao.DB.Create(&entity.Knowledgebase{
+ ID: "kb1",
+ TenantID: "tenant1",
+ Name: "Knowledge Base",
+ EmbdID: "embedding",
+ Permission: string(entity.TenantPermissionMe),
+ CreatedBy: "u1",
+ ParserID: "general",
+ Status: &status,
+ }).Error; err != nil {
+ t.Fatalf("create knowledgebase: %v", err)
+ }
+ if err := dao.DB.Create(&entity.User{
+ ID: "u1",
+ Nickname: "Tester",
+ Email: "[email protected]",
+ IsAuthenticated: "1",
+ IsActive: "1",
+ IsAnonymous: "0",
+ }).Error; err != nil {
+ t.Fatalf("create user: %v", err)
+ }
+
+ svc := NewFileCommitService()
+ commit, err := svc.RecordPageEdit(t.Context(), PageEditCommitInput{
+ DatasetID: "kb1",
+ Slug: "page-a",
+ PageType: "wiki",
+ Title: "Page A",
+ Comments: "published",
+ AuthorID: "u1",
+ OldContent: "old body",
+ NewContent: "new body",
+ })
+ if err != nil {
+ t.Fatalf("RecordPageEdit: %v", err)
+ }
+
+ detail, err := svc.GetPageCommitDetail(t.Context(), "kb1", commit.ID)
+ if err != nil {
+ t.Fatalf("GetPageCommitDetail: %v", err)
+ }
+ if detail.KBID != "kb1" || detail.TenantID != "tenant1" {
+ t.Fatalf("unexpected scope: %+v", detail)
+ }
+ if detail.Title != "Page A" || detail.Comments != "published" || detail.UserNickname != "Tester" {
+ t.Fatalf("unexpected metadata: %+v", detail)
+ }
+ if detail.ContentAfter != "new body" || !strings.Contains(detail.Diff, "new body") {
+ t.Fatalf("unexpected content: %+v", detail)
}
}
@@ -105,12 +169,12 @@ func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) {
ctx := t.Context()
base := PageEditCommitInput{
- DatasetID: "kb1",
- DocID: "wiki/page-a",
- Slug: "page-a",
- PageType: "wiki",
- Title: "first",
- AuthorID: "u1",
+ DatasetID: "kb1",
+ Slug: "page-a",
+ PageType: "wiki",
+ Title: "first",
+ AuthorID: "u1",
+ NewContent: "initial content",
}
if _, err := svc.RecordPageEdit(ctx, base); err != nil {
t.Fatalf("first RecordPageEdit: %v", err)
@@ -118,7 +182,7 @@ func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) {
second := base
second.Title = "second"
- second.OldContent = "hello"
+ second.OldContent = "initial content"
second.NewContent = "hello world"
commit2, err := svc.RecordPageEdit(ctx, second)
if err != nil {
@@ -136,6 +200,48 @@ func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) {
}
}
+func TestRecordPageEdit_FirstGeneratedVersionUsesAdd(t *testing.T) {
+ newPageCommitTestDB(t)
+ svc := NewFileCommitService()
+
+ commit, err := svc.RecordPageEdit(t.Context(), PageEditCommitInput{
+ DatasetID: "kb1",
+ Slug: "page-a",
+ PageType: "wiki",
+ OldContent: "",
+ NewContent: "initial content",
+ })
+ if err != nil {
+ t.Fatalf("RecordPageEdit: %v", err)
+ }
+ var item entity.FileCommitItem
+ if err := dao.DB.Where("commit_id = ?", commit.ID).First(&item).Error; err != nil {
+ t.Fatalf("load commit item: %v", err)
+ }
+ if item.Operation != "add" {
+ t.Fatalf("first generated version operation = %q, want add", item.Operation)
+ }
+}
+
+func TestRecordPageEdit_SkipsNoop(t *testing.T) {
+ newPageCommitTestDB(t)
+ svc := NewFileCommitService()
+
+ commit, err := svc.RecordPageEdit(t.Context(), PageEditCommitInput{
+ DatasetID: "kb1",
+ Slug: "page-a",
+ PageType: "wiki",
+ OldContent: "same",
+ NewContent: "same",
+ })
+ if err != nil {
+ t.Fatalf("RecordPageEdit: %v", err)
+ }
+ if commit != nil {
+ t.Fatalf("no-op edit created commit %s", commit.ID)
+ }
+}
+
func TestRecordPageEdit_IsolatesDatasets(t *testing.T) {
newPageCommitTestDB(t)
svc := NewFileCommitService()
@@ -144,7 +250,6 @@ func TestRecordPageEdit_IsolatesDatasets(t *testing.T) {
mk := func(datasetID string) PageEditCommitInput {
return PageEditCommitInput{
DatasetID: datasetID,
- DocID: datasetID + "/wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: datasetID + "-edit",
@@ -202,7 +307,6 @@ func TestRecordPageEdit_ConcurrentEditsFormLinearChain(t *testing.T) {
defer wg.Done()
in := PageEditCommitInput{
DatasetID: "kb1",
- DocID: "wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: "edit-" + strconv.Itoa(idx),
@@ -255,19 +359,19 @@ func TestUnifiedDiff_EmptyWhenNoChange(t *testing.T) {
func TestUnifiedDiff_DetectsAddition(t *testing.T) {
d := unifiedDiff("a\nb\nc\n", "a\nb\nc\nd\n")
- if !strings.Contains(d, "+d") {
+ if !strings.Contains(d, "@@") || !strings.Contains(d, "+d") {
t.Fatalf("expected diff to contain added line, got %q", d)
}
}
func TestUnifiedDiff_DetectsRemoval(t *testing.T) {
d := unifiedDiff("a\nb\nc\n", "a\nc\n")
- if !strings.Contains(d, "-b") {
+ if !strings.Contains(d, "@@") || !strings.Contains(d, "-b") {
t.Fatalf("expected diff to contain removed line, got %q", d)
}
}
-func TestUnifiedDiff_TruncatesLongDiff(t *testing.T) {
+func TestUnifiedDiff_UsesStandardHunkForLongDiff(t *testing.T) {
oldLines := make([]string, 50)
newLines := make([]string, 50)
for i := range oldLines {
@@ -275,8 +379,11 @@ func TestUnifiedDiff_TruncatesLongDiff(t *testing.T) {
newLines[i] = "new-line"
}
d := unifiedDiff(strings.Join(oldLines, "\n"), strings.Join(newLines, "\n"))
- if !strings.Contains(d, "... ") || !strings.Contains(d, "lines omitted") {
- t.Fatalf("expected long diff to be truncated, got %q", d)
+ if !strings.Contains(d, "--- a\n+++ b\n@@") {
+ t.Fatalf("expected standard unified diff headers, got %q", d)
+ }
+ if !strings.Contains(d, "-old-line") || !strings.Contains(d, "+new-line") {
+ t.Fatalf("expected removed and added lines, got %q", d)
}
}
@@ -300,7 +407,6 @@ func TestListPageCommits_ReturnsRecordedEdits(t *testing.T) {
base := PageEditCommitInput{
DatasetID: "kb1",
- DocID: "topic/fireworks display",
Slug: "fireworks display",
PageType: "topic",
AuthorID: "u1",
@@ -389,7 +495,6 @@ func TestListPageCommits_NestedSlugMatchesRecordPageEditKey(t *testing.T) {
// pageType="topic" and the full nested tail as the slug.
nested := PageEditCommitInput{
DatasetID: "kb1",
- DocID: "topic/People/Writers",
Slug: "People/Writers",
PageType: "topic",
AuthorID: "u1",
@@ -426,7 +531,6 @@ func TestListPageCommits_NestedSlugMatchesRecordPageEditKey(t *testing.T) {
// A page that only shares the first nested segment has its own history.
sibling := nested
- sibling.DocID = "topic/People/Editors"
sibling.Slug = "People/Editors"
sibling.Title = "sibling page"
siblingCommit, err := svc.RecordPageEdit(ctx, sibling)
@@ -476,7 +580,6 @@ func TestListPageCommits_ResolvesNicknamesInSingleBatchedLookup(t *testing.T) {
ctx := context.Background()
base := PageEditCommitInput{
DatasetID: "kb1",
- DocID: "topic/fireworks display",
Slug: "fireworks display",
PageType: "topic",
OldContent: "one",