Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(knowledge): add in-place file replacement primitive
ZHE MA419ff2507c25
diff --git a/internal/application/service/knowledge_replace.go b/internal/application/service/knowledge_replace.gonew file mode 100644index 000000000..fe99569f8--- /dev/null+++ b/internal/application/service/knowledge_replace.go@@ -0,0 +1,224 @@+package service++import (+ "context"+ "encoding/json"+ "mime/multipart"+ "time"++ werrors "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ secutils "github.com/Tencent/WeKnora/internal/utils"+)++// ReplaceKnowledgeFile swaps the source file of an existing file knowledge in+// place. The knowledge ID, and everything keyed on it (tags, references, data+// source metadata), is preserved while the stored file, hash, size, name and+// metadata are replaced and the document is re-parsed via ReparseKnowledge.+//+// Object storage is not transactional, so the steps are ordered to keep the+// row pointing at a file that exists:+// 1. save the new file under a fresh storage path;+// 2. point the row at it with a single UPDATE of the source columns;+// 3. ReparseKnowledge cleans the old chunks/index/graph and enqueues parsing;+// 4. only after that succeeds, delete the old file.+//+// If step 2 fails the new file is discarded. If step 3 fails the previous+// source columns are restored (marked failed, since cleanup may already have+// removed the old index) and the new file is discarded. A file is never+// deleted while the row may still reference it.+//+// Identical content under the same name and folder is not re-parsed: changed+// metadata is persisted and a DuplicateKnowledgeError carrying the knowledge+// is returned, mirroring CreateKnowledgeFromFile's duplicate contract.+func (s *knowledgeService) ReplaceKnowledgeFile(ctx context.Context,+ knowledgeID string, file *multipart.FileHeader, customFileName string, metadata map[string]string,+) (*types.Knowledge, error) {+ if file == nil {+ return nil, werrors.NewBadRequestError("file is required")+ }+ existing, kb, err := loadKnowledgeWrite(ctx, s.repo, s.kbService, knowledgeID)+ if err != nil {+ return nil, err+ }+ if existing.Type != "file" || existing.FilePath == "" {+ return nil, werrors.NewBadRequestError("only file knowledge can have its file replaced")+ }+ if existing.ParseStatus == types.ParseStatusDeleting {+ return nil, werrors.NewBadRequestError("knowledge is being deleted")+ }+ if err := s.checkStorageEngineConfigured(ctx, kb); err != nil {+ return nil, err+ }++ // Same naming rules as CreateKnowledgeFromFile: a path-qualified custom name+ // places the knowledge in that folder; without one the folder is kept.+ fileName, folderPath := file.Filename, existing.FolderPath+ if customFileName != "" {+ folderPath, fileName = types.SplitKnowledgeRelativePath(customFileName)+ if fileName == "" {+ fileName = file.Filename+ }+ }+ safeFileName, ok := secutils.ValidateInput(fileName)+ if !ok {+ return nil, werrors.NewValidationError("文件名包含非法字符")+ }+ if folderPath != "" {+ safeFolderPath, ok := secutils.ValidateInput(folderPath)+ if !ok {+ return nil, werrors.NewValidationError("文件夹路径包含非法字符")+ }+ folderPath = types.NormalizeKnowledgeFolderPath(safeFolderPath)+ }+ fileType := getFileType(safeFileName)++ // ReparseKnowledge reuses the stored overrides as-is, so check them against+ // the new file type here.+ overrides, err := existing.ProcessOverrides()+ if err != nil {+ return nil, err+ }+ if _, err := resolveFileImportProcessConfig(ctx, kb, fileType, overrides, nil); err != nil {+ return nil, err+ }++ hash, err := calculateFileHash(file)+ if err != nil {+ return nil, err+ }+ newMetadata, err := mergeKnowledgeMetadata(existing.Metadata, metadata)+ if err != nil {+ return nil, err+ }++ if hash == existing.FileHash && safeFileName == existing.FileName && folderPath == existing.FolderPath {+ current := existing.GetMetadata()+ for k, v := range metadata {+ if current[k] != v {+ if err := s.repo.UpdateKnowledgeColumn(ctx, existing.ID, "metadata", newMetadata); err != nil {+ return nil, err+ }+ existing.Metadata = newMetadata+ break+ }+ }+ return existing, types.NewDuplicateFileError(existing)+ }++ if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil &&+ tenant.StorageQuota > 0 && tenant.StorageUsed >= tenant.StorageQuota {+ return nil, types.NewStorageQuotaExceededError()+ }++ fileSvc := s.resolveFileService(ctx, kb)+ newPath, err := fileSvc.SaveFile(ctx, file, existing.TenantID, existing.ID)+ if err != nil {+ logger.Errorf(ctx, "Failed to save replacement file for knowledge %s: %v", existing.ID, err)+ return nil, err+ }+ // Compensations must still run if the caller's context is cancelled+ // mid-replacement (e.g. a sync task hitting its timeout).+ cleanupCtx := context.WithoutCancel(ctx)++ title := existing.Title+ if title == existing.FileName {+ title = safeFileName+ }+ previousMetadata := existing.Metadata+ if len(previousMetadata) == 0 {+ previousMetadata = types.JSON("{}") // metadata is NOT NULL; an empty JSON writes NULL+ }+ if err := s.repo.UpdateKnowledgeColumns(ctx, existing.ID, map[string]interface{}{+ "title": title,+ "file_name": safeFileName,+ "folder_path": folderPath,+ "file_type": fileType,+ "file_size": file.Size,+ "file_hash": hash,+ "file_path": newPath,+ "metadata": newMetadata,+ "updated_at": time.Now(),+ }); err != nil {+ logger.Errorf(ctx, "Failed to point knowledge %s at its replacement file: %v", existing.ID, err)+ s.discardReplacementFile(cleanupCtx, fileSvc, existing.TenantID, existing.ID, newPath)+ return nil, err+ }++ reparsed, err := s.ReparseKnowledge(ctx, existing.ID, nil)+ if err != nil {+ logger.Errorf(ctx, "Reparse after replacing the file of knowledge %s failed, restoring source: %v",+ existing.ID, err)+ if rerr := s.repo.UpdateKnowledgeColumns(cleanupCtx, existing.ID, map[string]interface{}{+ "title": existing.Title,+ "file_name": existing.FileName,+ "folder_path": existing.FolderPath,+ "file_type": existing.FileType,+ "file_size": existing.FileSize,+ "file_hash": existing.FileHash,+ "file_path": existing.FilePath,+ "metadata": previousMetadata,+ "parse_status": types.ParseStatusFailed,+ "error_message": "File replacement failed; reparse to rebuild the index",+ "updated_at": time.Now(),+ }); rerr != nil {+ logger.Errorf(ctx, "Failed to restore the source of knowledge %s: %v", existing.ID, rerr)+ }+ s.discardReplacementFile(cleanupCtx, fileSvc, existing.TenantID, existing.ID, newPath)+ return nil, err+ }++ if existing.FilePath != newPath {+ if err := fileSvc.DeleteFile(cleanupCtx, existing.FilePath); err != nil {+ logger.Warnf(ctx, "Failed to delete replaced file %s of knowledge %s: %v",+ existing.FilePath, existing.ID, err)+ }+ }+ recordKBActivity(ctx, s.audit, existing.TenantID, existing.KnowledgeBaseID, types.AuditActionKnowledgeUpdated,+ "knowledge", existing.ID, types.AuditOutcomeAccepted, map[string]any{+ "title": title, "source_type": "file", "file_type": fileType,+ "processing_status": "pending", "trigger": kbActivityTrigger(ctx),+ })+ return reparsed, nil+}++// discardReplacementFile deletes a replacement file that did not become the+// knowledge's source. It re-reads the row first: a write reported as failed+// may still have committed, and a knowledge pointing at a deleted file is+// worse than an orphaned blob, so the file is kept whenever the row still+// (or possibly) references it.+func (s *knowledgeService) discardReplacementFile(ctx context.Context, fileSvc interfaces.FileService,+ tenantID uint64, knowledgeID, filePath string,+) {+ current, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)+ if err != nil || (current != nil && current.FilePath == filePath) {+ logger.Warnf(ctx, "Keeping replacement file %s: knowledge %s may still reference it (lookup error: %v)",+ filePath, knowledgeID, err)+ return+ }+ if err := fileSvc.DeleteFile(ctx, filePath); err != nil {+ logger.Warnf(ctx, "Failed to delete discarded replacement file %s: %v", filePath, err)+ }+}++// mergeKnowledgeMetadata overlays updates on the stored metadata object, so+// entries the caller does not manage (e.g. process overrides) survive.+func mergeKnowledgeMetadata(current types.JSON, updates map[string]string) (types.JSON, error) {+ merged, err := current.Map()+ if err != nil {+ return nil, err+ }+ if merged == nil { // stored JSON null+ merged = map[string]interface{}{}+ }+ for k, v := range updates {+ merged[k] = v+ }+ b, err := json.Marshal(merged)+ if err != nil {+ return nil, err+ }+ return types.JSON(b), nil+}diff --git a/internal/application/service/knowledge_replace_test.go b/internal/application/service/knowledge_replace_test.gonew file mode 100644index 000000000..8ecf283bb--- /dev/null+++ b/internal/application/service/knowledge_replace_test.go@@ -0,0 +1,345 @@+package service++import (+ "context"+ "crypto/md5"+ "encoding/json"+ "errors"+ "fmt"+ "mime/multipart"+ "testing"++ "github.com/Tencent/WeKnora/internal/application/access"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/hibiken/asynq"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// replaceFileRepo is a single-row knowledge store. UpdateKnowledgeColumns can+// fail on a chosen call, either before applying the write or after it (a write+// that committed but reported an error).+type replaceFileRepo struct {+ interfaces.KnowledgeRepository+ row types.Knowledge+ columnsCalls int+ failColumnsCall int+ commitThenFail bool+}++func (r *replaceFileRepo) GetKnowledgeByID(context.Context, uint64, string) (*types.Knowledge, error) {+ row := r.row+ return &row, nil+}++func (r *replaceFileRepo) UpdateKnowledge(_ context.Context, knowledge *types.Knowledge) error {+ r.row = *knowledge+ return nil+}++func (r *replaceFileRepo) UpdateKnowledgeColumn(_ context.Context, _ string, column string, value interface{}) error {+ r.apply(map[string]interface{}{column: value})+ return nil+}++func (r *replaceFileRepo) UpdateKnowledgeColumns(_ context.Context, _ string, values map[string]interface{}) error {+ r.columnsCalls+++ if r.columnsCalls != r.failColumnsCall {+ r.apply(values)+ return nil+ }+ if r.commitThenFail {+ r.apply(values)+ }+ return errors.New("database unavailable")+}++func (r *replaceFileRepo) apply(values map[string]interface{}) {+ for column, value := range values {+ switch column {+ case "title":+ r.row.Title = value.(string)+ case "file_name":+ r.row.FileName = value.(string)+ case "folder_path":+ r.row.FolderPath = value.(string)+ case "file_type":+ r.row.FileType = value.(string)+ case "file_size":+ r.row.FileSize = value.(int64)+ case "file_hash":+ r.row.FileHash = value.(string)+ case "file_path":+ r.row.FilePath = value.(string)+ case "metadata":+ r.row.Metadata = value.(types.JSON)+ case "parse_status":+ r.row.ParseStatus = value.(string)+ case "error_message":+ r.row.ErrorMessage = value.(string)+ }+ }+}++type replaceFileStore struct {+ interfaces.FileService+ saveErr error+ saved int+ deleted []string+ events *[]string+}++func (f *replaceFileStore) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {+ if f.saveErr != nil {+ return "", f.saveErr+ }+ f.saved+++ *f.events = append(*f.events, "save")+ return "new/file.md", nil+}++func (f *replaceFileStore) DeleteFile(_ context.Context, filePath string) error {+ f.deleted = append(f.deleted, filePath)+ *f.events = append(*f.events, "delete:"+filePath)+ return nil+}++type replaceFileEnqueuer struct {+ err error+ payloads []types.DocumentProcessPayload+ events *[]string+}++func (e *replaceFileEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {+ if e.err != nil {+ return nil, e.err+ }+ var payload types.DocumentProcessPayload+ if err := json.Unmarshal(task.Payload(), &payload); err != nil {+ return nil, err+ }+ e.payloads = append(e.payloads, payload)+ *e.events = append(*e.events, "enqueue")+ return &asynq.TaskInfo{ID: "task-1", Queue: types.QueueDefault}, nil+}++type replaceFileChunks struct{ interfaces.ChunkRepository }++func (replaceFileChunks) ListImageInfoByKnowledgeIDs(+ context.Context, uint64, []string,+) ([]interfaces.ChunkImageInfo, error) {+ return nil, nil+}++func (replaceFileChunks) DeleteChunksByKnowledgeID(context.Context, uint64, string) error { return nil }++type replaceFileChunkService struct{ interfaces.ChunkService }++func (replaceFileChunkService) GetRepository() interfaces.ChunkRepository { return replaceFileChunks{} }++type replaceFileGraph struct {+ interfaces.RetrieveGraphRepository+}++func (replaceFileGraph) DelGraph(context.Context, []types.NameSpace) error { return nil }++type replaceFileHarness struct {+ svc *knowledgeService+ repo *replaceFileRepo+ store *replaceFileStore+ tasks *replaceFileEnqueuer+ events []string+ original types.Knowledge+ ctx context.Context+}++const replaceFileOldContent = "old"++func newReplaceFileHarness(t *testing.T) *replaceFileHarness {+ t.Helper()+ kb := &types.KnowledgeBase{ID: "kb-1", TenantID: 7}+ h := &replaceFileHarness{}+ h.original = types.Knowledge{+ ID: "knowledge-1",+ TenantID: 7,+ KnowledgeBaseID: "kb-1",+ Type: "file",+ Title: "a.md",+ FileName: "a.md",+ FolderPath: "notes",+ FileType: "md",+ FileSize: int64(len(replaceFileOldContent)),+ FileHash: md5Hex(replaceFileOldContent),+ FilePath: "old/file.md",+ ParseStatus: types.ParseStatusCompleted,+ EnableStatus: "enabled",+ Metadata: types.JSON(`{"external_id":"notes/a.md","extra":{"nested":true}}`),+ }+ h.repo = &replaceFileRepo{row: h.original}+ h.store = &replaceFileStore{events: &h.events}+ h.tasks = &replaceFileEnqueuer{events: &h.events}+ h.svc = &knowledgeService{+ repo: h.repo,+ kbService: &reparseFailureKBService{kb: kb},+ fileSvc: h.store,+ task: h.tasks,+ chunkService: replaceFileChunkService{},+ chunkRepo: replaceFileChunks{},+ graphEngine: replaceFileGraph{},+ }+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))+ ctx = context.WithValue(ctx, types.TenantInfoContextKey, &types.Tenant{ID: 7})+ ctx, err := access.WithKBTaskWrite(ctx, kb, 7)+ require.NoError(t, err)+ h.ctx = ctx+ return h+}++func (h *replaceFileHarness) replace(+ t *testing.T, content, customFileName string, metadata map[string]string,+) (*types.Knowledge, error) {+ t.Helper()+ fh, err := bytesToFileHeader([]byte(content), "upload.md")+ require.NoError(t, err)+ return h.svc.ReplaceKnowledgeFile(h.ctx, h.original.ID, fh, customFileName, metadata)+}++func md5Hex(content string) string {+ return fmt.Sprintf("%x", md5.Sum([]byte(content)))+}++func TestReplaceKnowledgeFilePreservesIDAndReparsesNewContent(t *testing.T) {+ h := newReplaceFileHarness(t)+ content := "# new body"++ got, err := h.replace(t, content, "notes/sub/b.md",+ map[string]string{"source_updated_at": "2026-09-14T00:00:00Z"})++ require.NoError(t, err)+ require.Equal(t, h.original.ID, got.ID)+ row := h.repo.row+ assert.Equal(t, "new/file.md", row.FilePath)+ assert.Equal(t, md5Hex(content), row.FileHash)+ assert.Equal(t, int64(len(content)), row.FileSize)+ assert.Equal(t, "b.md", row.FileName)+ assert.Equal(t, "b.md", row.Title, "a title that mirrored the file name follows the rename")+ assert.Equal(t, "notes/sub", row.FolderPath)+ assert.Equal(t, types.ParseStatusPending, row.ParseStatus)++ metadata, err := row.Metadata.Map()+ require.NoError(t, err)+ assert.Equal(t, "notes/a.md", metadata["external_id"])+ assert.Equal(t, "2026-09-14T00:00:00Z", metadata["source_updated_at"])+ assert.Equal(t, map[string]interface{}{"nested": true}, metadata["extra"], "unmanaged entries survive")++ require.Len(t, h.tasks.payloads, 1)+ assert.Equal(t, h.original.ID, h.tasks.payloads[0].KnowledgeID)+ assert.Equal(t, "new/file.md", h.tasks.payloads[0].FilePath)+ assert.Equal(t, []string{"save", "enqueue", "delete:old/file.md"}, h.events,+ "the old file is deleted only after the reparse was submitted")+}++func TestReplaceKnowledgeFileUnchangedContentSkipsReparse(t *testing.T) {+ h := newReplaceFileHarness(t)++ got, err := h.replace(t, replaceFileOldContent, "notes/a.md", map[string]string{"source_updated_at": "later"})++ var dupErr *types.DuplicateKnowledgeError+ require.ErrorAs(t, err, &dupErr)+ require.Equal(t, h.original.ID, got.ID)+ assert.Equal(t, h.original.ID, dupErr.Knowledge.ID)+ assert.Zero(t, h.store.saved)+ assert.Empty(t, h.tasks.payloads)+ assert.Empty(t, h.store.deleted)+ assert.Equal(t, h.original.FilePath, h.repo.row.FilePath)+ assert.Equal(t, "later", h.repo.row.GetMetadata()["source_updated_at"], "changed metadata is still persisted")+}++func TestReplaceKnowledgeFileSaveFailureLeavesKnowledgeUntouched(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.store.saveErr = errors.New("storage unavailable")++ _, err := h.replace(t, "# new body", "notes/a.md", nil)++ require.ErrorIs(t, err, h.store.saveErr)+ assert.Equal(t, h.original, h.repo.row)+ assert.Empty(t, h.store.deleted)+ assert.Empty(t, h.tasks.payloads)+}++func TestReplaceKnowledgeFileSourceUpdateFailureDiscardsNewFile(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.repo.failColumnsCall = 1++ _, err := h.replace(t, "# new body", "notes/a.md", nil)++ require.Error(t, err)+ assert.Equal(t, h.original, h.repo.row)+ assert.Equal(t, []string{"new/file.md"}, h.store.deleted, "the old file must never be deleted")+ assert.Empty(t, h.tasks.payloads)+}++func TestReplaceKnowledgeFileCommittedUpdateReportedAsFailedKeepsNewFile(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.repo.failColumnsCall = 1+ h.repo.commitThenFail = true++ _, err := h.replace(t, "# new body", "notes/a.md", nil)++ require.Error(t, err)+ assert.Equal(t, "new/file.md", h.repo.row.FilePath)+ assert.Empty(t, h.store.deleted, "a file the row references must not be deleted")+}++func TestReplaceKnowledgeFileReparseFailureRestoresPreviousSource(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.tasks.err = errors.New("queue unavailable")++ _, err := h.replace(t, "# new body", "notes/sub/b.md", map[string]string{"source_updated_at": "later"})++ require.Error(t, err)+ row := h.repo.row+ assert.Equal(t, h.original.ID, row.ID)+ assert.Equal(t, h.original.FilePath, row.FilePath)+ assert.Equal(t, h.original.FileHash, row.FileHash)+ assert.Equal(t, h.original.FileSize, row.FileSize)+ assert.Equal(t, h.original.FileName, row.FileName)+ assert.Equal(t, h.original.Title, row.Title)+ assert.Equal(t, h.original.FolderPath, row.FolderPath)+ assert.JSONEq(t, string(h.original.Metadata), string(row.Metadata))+ assert.Equal(t, types.ParseStatusFailed, row.ParseStatus, "the old index may already be cleaned up")+ assert.Equal(t, []string{"new/file.md"}, h.store.deleted)+}++func TestReplaceKnowledgeFileRestoreFailureKeepsBothFiles(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.tasks.err = errors.New("queue unavailable")+ h.repo.failColumnsCall = 2 // the restore write++ _, err := h.replace(t, "# new body", "notes/a.md", nil)++ require.Error(t, err)+ assert.Equal(t, "new/file.md", h.repo.row.FilePath)+ assert.Empty(t, h.store.deleted)+}++func TestReplaceKnowledgeFileRejectsNonFileKnowledge(t *testing.T) {+ h := newReplaceFileHarness(t)+ h.repo.row.Type = types.KnowledgeTypeManual++ _, err := h.replace(t, "# new body", "notes/a.md", nil)++ require.Error(t, err)+ assert.Zero(t, h.store.saved)+}++func TestReplaceKnowledgeFileRejectsUnsupportedFileType(t *testing.T) {+ h := newReplaceFileHarness(t)++ _, err := h.replace(t, "MZ", "notes/tool.exe", nil)++ require.Error(t, err)+ assert.Zero(t, h.store.saved)+ assert.Equal(t, h.original, h.repo.row)+}diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.goindex 88ceba3f5..0ed7b91a4 100644--- a/internal/types/interfaces/knowledge.go+++ b/internal/types/interfaces/knowledge.go@@ -128,6 +128,17 @@ type KnowledgeService interface {knowledgeID string,processOverrides *types.KnowledgeProcessOverrides,) (*types.Knowledge, error)+ // ReplaceKnowledgeFile replaces the source file of an existing file knowledge+ // while preserving its ID, then re-parses it. A path-qualified customFileName+ // also sets the folder; metadata entries are merged into the stored metadata.+ // Identical content returns a DuplicateKnowledgeError without re-parsing.+ ReplaceKnowledgeFile(+ ctx context.Context,+ knowledgeID string,+ file *multipart.FileHeader,+ customFileName string,+ metadata map[string]string,+ ) (*types.Knowledge, error)// CancelKnowledgeParse marks an in-progress parse as cancelled by the// user. The knowledge row and any partially written chunks/index are// kept; downstream queued tasks for the same knowledge are best-effort