Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix: preserve shared resources during clone rollback (#4623)
Lucienbc1ade36b933
diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.goindex b8025158a..5175028be 100644--- a/internal/application/service/knowledge_clone_move.go+++ b/internal/application/service/knowledge_clone_move.go@@ -24,8 +24,8 @@ import ("github.com/redis/go-redis/v9")-// copyOwnedObject copies srcPath into a NEW object owned by the destination-// tenant, returning the new provider:// (resource) path.+// copyOwnedObject copies srcPath for the destination tenant and claims the+// returned resource for its knowledge. Identical images may reuse a catalog blob.//// Extracted/embedded chunk images MUST land in the tenant's exports/ namespace,// because GET /knowledge-bases/:id/files only serves objects that pass@@ -35,7 +35,7 @@ import (// produced images that could no longer be rendered. Instead, read the source// bytes and re-save them via SaveBytes, exactly mirroring how the original// images were persisted during ingestion (see image_resolver.saveReferencedImage),-// so the copy is a genuine independent object in the servable namespace.+// so the copy is in the servable namespace.func copyOwnedObject(ctx context.Context,srcSvc, dstSvc interfaces.FileService,@@ -122,8 +122,8 @@ func isImageExt(ext string) bool {}// cloneChunkImageInfo parses a chunk's image_info JSON, copies every referenced-// object into a NEW object owned by (tenantID, knowledgeID), and returns the-// re-serialized image_info plus the list of newly-created object URLs (for+// object for (tenantID, knowledgeID), and returns the re-serialized image_info+// plus copied resource URLs (including reused images, for ownership-aware// rollback on failure). urlCache dedups identical source objects across chunks// so the same source image is copied at most once per clone AND accumulates the// full old->new URL mapping so callers can rewrite in-content Markdown image@@ -213,22 +213,13 @@ func rewriteContentImageURLs(content string, urlCache map[string]string) stringreturn content}-// cleanupCopiedObjects deletes objects that were newly created during a clone-// that subsequently failed, to avoid orphaning storage. It is best-effort:-// delete errors are logged but never returned (the original clone error wins).-func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths []string) {+// cleanupCopiedObjects releases a failed clone's claims before deleting any+// unreferenced bytes. SaveBytes may have reused another owner's image.+func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, releaser *resourceReleaser, paths []string) {if len(paths) == 0 || svc == nil {return}- logger.Infof(ctx, "Cleaning up %d copied objects after clone failure", len(paths))- for _, p := range paths {- if p == "" {- continue- }- if err := svc.DeleteFile(ctx, p); err != nil {- logger.Errorf(ctx, "Failed to clean up copied object %s: %v", p, err)- }- }+ deleteExtractedImages(ctx, svc, releaser, paths)}func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error {@@ -339,7 +330,10 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowlreturn}}- cleanupCopiedObjects(ctx, dstSvc, copiedURLs)+ // The source file is already owned by this destination before chunk+ // cloning. An identical embedded image must not release that claim.+ copiedURLs = slices.DeleteFunc(copiedURLs, func(ref string) bool { return ref == stored.FilePath })+ cleanupCopiedObjects(ctx, dstSvc, knowledgeResourceOwners(s.resourceCatalog, dst.ID), copiedURLs)}()now := time.Now()@@ -673,12 +667,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(// new objects for best-effort cleanup if the clone fails partway through.dstSvc := s.resolveFileService(ctx, dstKB)imageURLCache := map[string]string{}- var copiedImageURLs []string- defer func() {- if retErr != nil {- cleanupCopiedObjects(ctx, dstSvc, copiedImageURLs)- }- }()// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)@@ -831,6 +819,42 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(handleError(progress, err, "Failed to prepare FAQ knowledge entry")return err}+ // Bind is idempotent per knowledge, so rollback must preserve claims that+ // predate this incremental clone as well as images in persisted batches.+ retainedImages := map[string]bool{}+ if s.resourceCatalog != nil {+ refs, err := s.resourceCatalog.ListReferencesByOwner(ctx, types.ResourceOwnerKnowledge, dstKnowledge.ID)+ if err != nil {+ handleError(progress, err, "Failed to load existing FAQ resource claims")+ return fmt.Errorf("load existing FAQ resource claims: %w", err)+ }+ for _, ref := range refs {+ retainedImages[ref] = true+ }+ }+ var copiedImageURLs []string+ defer func() {+ if retErr == nil || len(copiedImageURLs) == 0 {+ return+ }+ // A failed CreateChunks may have committed before its acknowledgement+ // was lost. Inspect persisted rows, keeping all bytes on an uncertain read.+ rows, err := s.chunkRepo.ListAllChunksByKnowledgeID(ctx, dstKB.TenantID, dstKnowledge.ID)+ if err != nil {+ logger.Warnf(ctx, "Keeping copied FAQ images: failed to read saved chunks: %v", err)+ return+ }+ for _, row := range rows {+ for _, ref := range collectImageURLs(ctx, []string{row.ImageInfo}) {+ retainedImages[ref] = true+ }+ for _, ref := range types.ScanResourceReferences(row.Content) {+ retainedImages[ref] = true+ }+ }+ copiedImageURLs = slices.DeleteFunc(copiedImageURLs, func(ref string) bool { return retainedImages[ref] })+ cleanupCopiedObjects(ctx, dstSvc, knowledgeResourceOwners(s.resourceCatalog, dstKnowledge.ID), copiedImageURLs)+ }()// Clone FAQ chunks from source to destinationbatch := 50@@ -859,13 +883,13 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(// FAQ knowledge so deleting the source never breaks this clone.newImageInfo, copied, copyErr := cloneChunkImageInfo(ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache, s.resourceCatalog)+ copiedImageURLs = append(copiedImageURLs, copied...)if copyErr != nil {logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)handleError(progress, copyErr, "Failed to copy FAQ entry images")retErr = copyErrreturn retErr}- copiedImageURLs = append(copiedImageURLs, copied...)newChunk := &types.Chunk{ID: uuid.New().String(),diff --git a/internal/application/service/knowledge_clone_rollback_test.go b/internal/application/service/knowledge_clone_rollback_test.gonew file mode 100644index 000000000..8128be5a7--- /dev/null+++ b/internal/application/service/knowledge_clone_rollback_test.go@@ -0,0 +1,291 @@+package service++import (+ "context"+ "crypto/sha256"+ "errors"+ "fmt"+ "io"+ "testing"++ "github.com/Tencent/WeKnora/internal/application/access"+ filesvc "github.com/Tencent/WeKnora/internal/application/service/file"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++func cloneImageHash(data []byte) string {+ return fmt.Sprintf("%x", sha256.Sum256(data))+}++func cloneRollbackStorage(t *testing.T, f *documentWriteFixture) (interfaces.ResourceCatalog, interfaces.FileService) {+ t.Helper()+ catalog, _ := newResourceCatalogForTest(t)+ physical := filesvc.NewLocalFileService(t.TempDir(), "")+ f.svc.resourceCatalog = catalog+ f.svc.fileSvc = filesvc.NewResourceCatalogFileService(physical, catalog)+ return catalog, physical+}++func requireCloneImageReadable(ctx context.Context, t *testing.T, files interfaces.FileService, ref string) {+ t.Helper()+ r, err := files.GetFile(ctx, ref)+ require.NoError(t, err, "image %s must survive a failed clone", ref)+ defer func() { require.NoError(t, r.Close()) }()+ data, err := io.ReadAll(r)+ require.NoError(t, err)+ require.NotEmpty(t, data)+}++func TestCloneRollbackRespectsResourceOwnership(t *testing.T) {+ for _, failDuringCopy := range []bool{false, true} {+ t.Run(fmt.Sprintf("image copy failure=%t", failDuringCopy), func(t *testing.T) {+ t.Setenv("RETRIEVE_DRIVER", "postgres")+ f := transferFixture(t, access.KBTransferClone)+ catalog, physical := cloneRollbackStorage(t, f)+ shared, err := f.svc.fileSvc.SaveBytes(f.ctx, []byte("shared image"), 7, "shared.png", false)+ require.NoError(t, err)+ require.NoError(t, catalog.Bind(+ f.ctx, shared, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationExtractedImage))+ legacy, err := physical.SaveBytes(f.ctx, []byte("legacy image"), 7, "legacy.png", false)+ require.NoError(t, err)+ images := []types.ImageInfo{{URL: shared}, {URL: legacy}}+ if failDuringCopy {+ images = append(images, types.ImageInfo{URL: "local://7/exports/missing.png"})+ }+ require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").+ Update("image_info", mustImageInfoJSON(t, images)).Error)+ f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: &partialCloneEngine{vectors: map[string]int{}}}+ f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}}+ f.kbs.values["other"].EmbeddingModelID = "model"+ src, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc")+ require.NoError(t, err)+ err = f.svc.cloneKnowledge(f.ctx, src, f.kbs.values["other"])+ if failDuringCopy {+ require.ErrorContains(t, err, "failed to copy chunk image")+ } else {+ require.ErrorContains(t, err, "partial vector copy")+ }+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, shared)+ requireCloneImageReadable(f.ctx, t, physical, legacy)+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, "doc")+ require.NoError(t, err)+ require.Equal(t, []string{shared}, refs)+ rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other")+ require.NoError(t, err)+ for _, dst := range rows {+ if dst.ID == "other-doc" {+ continue+ }+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, dst.ID)+ require.NoError(t, err)+ require.Empty(t, refs, "failed destination must release its claims")+ }+ // The legacy image needed a new catalog resource. Only the shared+ // source should remain eligible for content-hash reuse after rollback.+ ref, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("legacy image")))+ require.NoError(t, err)+ require.False(t, reused, "new destination-only resource must be reclaimed: %s", ref)+ })+ }+}++type cloneCreateFailureRepo struct{ interfaces.KnowledgeRepository }++func (cloneCreateFailureRepo) CreateKnowledge(context.Context, *types.Knowledge) error {+ return errors.New("create clone failed")+}++func TestCloneRollbackPreservesImageSourceFile(t *testing.T) {+ for _, failedCreate := range []bool{false, true} {+ t.Run(fmt.Sprintf("create failure=%t", failedCreate), func(t *testing.T) {+ f := transferFixture(t, access.KBTransferClone)+ catalog, _ := cloneRollbackStorage(t, f)+ ref, err := f.svc.fileSvc.SaveBytes(f.ctx, []byte("image source file"), 7, "source.png", false)+ require.NoError(t, err)+ require.NoError(t, catalog.Bind(+ f.ctx, ref, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationSourceFile))+ require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("file_path", ref).Error)+ images := []types.ImageInfo{{URL: ref}, {URL: "local://7/exports/missing.png"}}+ require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").+ Update("image_info", mustImageInfoJSON(t, images)).Error)+ if failedCreate {+ f.svc.repo = cloneCreateFailureRepo{f.repo}+ }+ src, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc")+ require.NoError(t, err)+ require.Error(t, f.svc.cloneKnowledge(f.ctx, src, f.kbs.values["other"]))+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, ref)+ rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other")+ require.NoError(t, err)+ for _, dst := range rows {+ if dst.ID == "other-doc" {+ continue+ }+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, dst.ID)+ require.NoError(t, err)+ require.Equal(t, []string{ref}, refs, "persisted destination still owns its source file")+ }+ })+ }+}++// The real repositories and storage run normally; only the selected CreateChunks+// acknowledgement is injected, so the second FAQ batch has deterministic failure.+type faqCloneRollbackChunks struct {+ interfaces.ChunkRepository+ ids []string+ creates int+ failCreate int+ commit bool+ failRead bool+}++func (r *faqCloneRollbackChunks) ListAllChunksByKnowledgeID(+ ctx context.Context, tenant uint64, id string,+) ([]*types.Chunk, error) {+ if r.failRead && r.creates > 0 {+ return nil, errors.New("read saved FAQ chunks failed")+ }+ return r.ChunkRepository.ListAllChunksByKnowledgeID(ctx, tenant, id)+}++type cloneRollbackCatalogFault struct {+ interfaces.ResourceCatalog+ failList bool+}++func (c cloneRollbackCatalogFault) ListReferencesByOwner(+ ctx context.Context, owner string, ids ...string,+) ([]string, error) {+ if c.failList {+ return nil, errors.New("read resource claims failed")+ }+ return c.ResourceCatalog.ListReferencesByOwner(ctx, owner, ids...)+}++func (cloneRollbackCatalogFault) Release(context.Context, string, string, string) (int64, error) {+ return 0, errors.New("release resource failed")+}++func (r *faqCloneRollbackChunks) FAQChunkDiff(+ context.Context, uint64, string, uint64, string,+) (*types.FAQChunkDiffResult, error) {+ return &types.FAQChunkDiffResult{ChunksToAdd: r.ids}, nil+}++func (r *faqCloneRollbackChunks) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {+ r.creates+++ if r.creates == r.failCreate {+ if r.commit {+ if err := r.ChunkRepository.CreateChunks(ctx, chunks); err != nil {+ return err+ }+ }+ return errors.New("FAQ batch write failed")+ }+ return r.ChunkRepository.CreateChunks(ctx, chunks)+}++func TestFAQCloneRollbackKeepsSavedImagesAndReclaimsPartialCopies(t *testing.T) {+ for _, scenario := range []string{+ "partial image copy", "saved batch", "existing destination", "lost acknowledgement",+ "uncertain saved rows", "release fails", "owner lookup fails",+ } {+ t.Run(scenario, func(t *testing.T) {+ t.Setenv("RETRIEVE_DRIVER", "postgres")+ f := transferFixture(t, access.KBTransferClone)+ catalog, physical := cloneRollbackStorage(t, f)+ for _, kb := range f.kbs.values {+ kb.Type = types.KnowledgeBaseTypeFAQ+ }+ require.NoError(t, f.db.Model(&types.Knowledge{}).Where("tenant_id = ?", 7).+ Update("type", types.KnowledgeTypeFAQ).Error)+ require.NoError(t, f.db.Where("tenant_id = ?", 7).Delete(&types.Chunk{}).Error)+ f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: &cloneCopyEngine{}}+ f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}}+ first, err := physical.SaveBytes(f.ctx, []byte("saved image"), 7, "first.png", false)+ require.NoError(t, err)+ alias, err := physical.SaveBytes(f.ctx, []byte("saved image"), 7, "alias.png", false)+ require.NoError(t, err)+ exclusive, err := physical.SaveBytes(f.ctx, []byte("pending image"), 7, "pending.png", false)+ require.NoError(t, err)+ var saved string+ if scenario == "existing destination" {+ saved, err = f.svc.fileSvc.SaveBytes(f.ctx, []byte("saved image"), 7, "existing.png", false)+ require.NoError(t, err)+ require.NoError(t, catalog.Bind(+ f.ctx, saved, types.ResourceOwnerKnowledge, "other-doc", types.ResourceRelationAttachment))+ }+ fault := &faqCloneRollbackChunks{+ ChunkRepository: f.chunkRepo, failCreate: 1, commit: scenario == "lost acknowledgement",+ failRead: scenario == "uncertain saved rows",+ }+ if scenario == "release fails" || scenario == "owner lookup fails" {+ f.svc.resourceCatalog = cloneRollbackCatalogFault{catalog, scenario == "owner lookup fails"}+ }+ if scenario == "saved batch" {+ fault.failCreate = 2+ for i := 0; i < 50; i++ {+ id := fmt.Sprintf("saved-%02d", i)+ fault.ids = append(fault.ids, id)+ require.NoError(t, f.db.Create(&types.Chunk{+ ID: id, TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ,+ ImageInfo: mustImageInfoJSON(t, []types.ImageInfo{{URL: first}}),+ Metadata: types.JSON(`{"standard_question":"question","answers":["answer"]}`),+ }).Error)+ }+ }+ images := []types.ImageInfo{{URL: alias}, {URL: exclusive}}+ if scenario == "partial image copy" {+ images = append(images, types.ImageInfo{URL: "local://7/exports/missing.png"})+ }+ fault.ids = append(fault.ids, "pending")+ require.NoError(t, f.db.Create(&types.Chunk{+ ID: "pending", TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ,+ ImageInfo: mustImageInfoJSON(t, images),+ Metadata: types.JSON(`{"standard_question":"question","answers":["answer"]}`),+ }).Error)+ f.svc.chunkRepo = fault+ handledError := false+ err = f.svc.cloneFAQKnowledgeBase(+ f.ctx, f.kbs.values["kb"], f.kbs.values["other"], &types.KBCloneProgress{TaskID: "clone"},+ func(*types.KBCloneProgress, error, string) { handledError = true },+ )+ require.Error(t, err)+ require.True(t, handledError, "clone progress must report the failure")+ if scenario == "owner lookup fails" {+ require.ErrorContains(t, err, "read resource claims failed")+ require.Zero(t, fault.creates, "do not copy or write without knowing existing claims")+ _, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("pending image")))+ require.NoError(t, err)+ require.False(t, reused)+ return+ }+ if scenario == "partial image copy" {+ require.ErrorContains(t, err, "failed to copy chunk image")+ } else {+ require.ErrorContains(t, err, "FAQ batch write failed")+ }+ chunks, err := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, "other-doc")+ require.NoError(t, err)+ for _, chunk := range chunks {+ for _, ref := range collectImageURLs(f.ctx, []string{chunk.ImageInfo}) {+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, ref)+ }+ }+ if saved != "" {+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, saved)+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, "other-doc")+ require.NoError(t, err)+ require.Equal(t, []string{saved}, refs)+ }+ _, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("pending image")))+ require.NoError(t, err)+ keep := scenario == "lost acknowledgement" || scenario == "uncertain saved rows" ||+ scenario == "release fails"+ require.Equal(t, keep, reused, "keep saved images and fail closed when ownership is uncertain")+ })+ }+}diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 7ffd22f00..c0e697714 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -114,7 +114,10 @@ func (s *knowledgeService) cloneKnowledge(}stored, loadErr := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID)if errors.Is(loadErr, repository.ErrKnowledgeNotFound) {- cleanupCopiedObjects(ctx, s.resolveFileService(ctx, targetKB), copiedFilePaths)+ cleanupCopiedObjects(+ ctx, s.resolveFileService(ctx, targetKB),+ knowledgeResourceOwners(s.resourceCatalog, dst.ID), copiedFilePaths,+ )return}// An uncertain save/read must not delete objects that may be referenced.