Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix: preserve shared resources during clone rollback (#4623)

Lucienbc1ade36b933
diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go
index b8025158a..5175028be 100644
--- a/internal/application/service/knowledge_clone_move.go
+++ b/internal/application/service/knowledge_clone_move.go
@@ -24,8 +24,8 @@ import (
"github.com/redis/go-redis/v9"
)
-// copyOwnedObject copies srcPath into a NEW object owned by the destination
-// tenant, returning the new provider:// (resource) path.
+// copyOwnedObject copies srcPath for the destination tenant and claims the
+// returned resource for its knowledge. Identical images may reuse a catalog blob.
//
// Extracted/embedded chunk images MUST land in the tenant's exports/ namespace,
// because GET /knowledge-bases/:id/files only serves objects that pass
@@ -35,7 +35,7 @@ import (
// produced images that could no longer be rendered. Instead, read the source
// bytes and re-save them via SaveBytes, exactly mirroring how the original
// images were persisted during ingestion (see image_resolver.saveReferencedImage),
-// so the copy is a genuine independent object in the servable namespace.
+// so the copy is in the servable namespace.
func copyOwnedObject(
ctx context.Context,
srcSvc, dstSvc interfaces.FileService,
@@ -122,8 +122,8 @@ func isImageExt(ext string) bool {
}
// cloneChunkImageInfo parses a chunk's image_info JSON, copies every referenced
-// object into a NEW object owned by (tenantID, knowledgeID), and returns the
-// re-serialized image_info plus the list of newly-created object URLs (for
+// object for (tenantID, knowledgeID), and returns the re-serialized image_info
+// plus copied resource URLs (including reused images, for ownership-aware
// rollback on failure). urlCache dedups identical source objects across chunks
// so the same source image is copied at most once per clone AND accumulates the
// full old->new URL mapping so callers can rewrite in-content Markdown image
@@ -213,22 +213,13 @@ func rewriteContentImageURLs(content string, urlCache map[string]string) string
return content
}
-// cleanupCopiedObjects deletes objects that were newly created during a clone
-// that subsequently failed, to avoid orphaning storage. It is best-effort:
-// delete errors are logged but never returned (the original clone error wins).
-func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, paths []string) {
+// cleanupCopiedObjects releases a failed clone's claims before deleting any
+// unreferenced bytes. SaveBytes may have reused another owner's image.
+func cleanupCopiedObjects(ctx context.Context, svc interfaces.FileService, releaser *resourceReleaser, paths []string) {
if len(paths) == 0 || svc == nil {
return
}
- logger.Infof(ctx, "Cleaning up %d copied objects after clone failure", len(paths))
- for _, p := range paths {
- if p == "" {
- continue
- }
- if err := svc.DeleteFile(ctx, p); err != nil {
- logger.Errorf(ctx, "Failed to clean up copied object %s: %v", p, err)
- }
- }
+ deleteExtractedImages(ctx, svc, releaser, paths)
}
func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error {
@@ -339,7 +330,10 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
return
}
}
- cleanupCopiedObjects(ctx, dstSvc, copiedURLs)
+ // The source file is already owned by this destination before chunk
+ // cloning. An identical embedded image must not release that claim.
+ copiedURLs = slices.DeleteFunc(copiedURLs, func(ref string) bool { return ref == stored.FilePath })
+ cleanupCopiedObjects(ctx, dstSvc, knowledgeResourceOwners(s.resourceCatalog, dst.ID), copiedURLs)
}()
now := time.Now()
@@ -673,12 +667,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
// new objects for best-effort cleanup if the clone fails partway through.
dstSvc := s.resolveFileService(ctx, dstKB)
imageURLCache := map[string]string{}
- var copiedImageURLs []string
- defer func() {
- if retErr != nil {
- cleanupCopiedObjects(ctx, dstSvc, copiedImageURLs)
- }
- }()
// Get source FAQ knowledge first (FAQ KB has exactly one Knowledge entry)
srcKnowledgeList, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, srcKB.TenantID, srcKB.ID)
@@ -831,6 +819,42 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
handleError(progress, err, "Failed to prepare FAQ knowledge entry")
return err
}
+ // Bind is idempotent per knowledge, so rollback must preserve claims that
+ // predate this incremental clone as well as images in persisted batches.
+ retainedImages := map[string]bool{}
+ if s.resourceCatalog != nil {
+ refs, err := s.resourceCatalog.ListReferencesByOwner(ctx, types.ResourceOwnerKnowledge, dstKnowledge.ID)
+ if err != nil {
+ handleError(progress, err, "Failed to load existing FAQ resource claims")
+ return fmt.Errorf("load existing FAQ resource claims: %w", err)
+ }
+ for _, ref := range refs {
+ retainedImages[ref] = true
+ }
+ }
+ var copiedImageURLs []string
+ defer func() {
+ if retErr == nil || len(copiedImageURLs) == 0 {
+ return
+ }
+ // A failed CreateChunks may have committed before its acknowledgement
+ // was lost. Inspect persisted rows, keeping all bytes on an uncertain read.
+ rows, err := s.chunkRepo.ListAllChunksByKnowledgeID(ctx, dstKB.TenantID, dstKnowledge.ID)
+ if err != nil {
+ logger.Warnf(ctx, "Keeping copied FAQ images: failed to read saved chunks: %v", err)
+ return
+ }
+ for _, row := range rows {
+ for _, ref := range collectImageURLs(ctx, []string{row.ImageInfo}) {
+ retainedImages[ref] = true
+ }
+ for _, ref := range types.ScanResourceReferences(row.Content) {
+ retainedImages[ref] = true
+ }
+ }
+ copiedImageURLs = slices.DeleteFunc(copiedImageURLs, func(ref string) bool { return retainedImages[ref] })
+ cleanupCopiedObjects(ctx, dstSvc, knowledgeResourceOwners(s.resourceCatalog, dstKnowledge.ID), copiedImageURLs)
+ }()
// Clone FAQ chunks from source to destination
batch := 50
@@ -859,13 +883,13 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
// FAQ knowledge so deleting the source never breaks this clone.
newImageInfo, copied, copyErr := cloneChunkImageInfo(
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache, s.resourceCatalog)
+ copiedImageURLs = append(copiedImageURLs, copied...)
if copyErr != nil {
logger.Errorf(ctx, "Failed to copy FAQ chunk images: %v", copyErr)
handleError(progress, copyErr, "Failed to copy FAQ entry images")
retErr = copyErr
return retErr
}
- copiedImageURLs = append(copiedImageURLs, copied...)
newChunk := &types.Chunk{
ID: uuid.New().String(),
diff --git a/internal/application/service/knowledge_clone_rollback_test.go b/internal/application/service/knowledge_clone_rollback_test.go
new file mode 100644
index 000000000..8128be5a7
--- /dev/null
+++ b/internal/application/service/knowledge_clone_rollback_test.go
@@ -0,0 +1,291 @@
+package service
+
+import (
+ "context"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "io"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/application/access"
+ filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func cloneImageHash(data []byte) string {
+ return fmt.Sprintf("%x", sha256.Sum256(data))
+}
+
+func cloneRollbackStorage(t *testing.T, f *documentWriteFixture) (interfaces.ResourceCatalog, interfaces.FileService) {
+ t.Helper()
+ catalog, _ := newResourceCatalogForTest(t)
+ physical := filesvc.NewLocalFileService(t.TempDir(), "")
+ f.svc.resourceCatalog = catalog
+ f.svc.fileSvc = filesvc.NewResourceCatalogFileService(physical, catalog)
+ return catalog, physical
+}
+
+func requireCloneImageReadable(ctx context.Context, t *testing.T, files interfaces.FileService, ref string) {
+ t.Helper()
+ r, err := files.GetFile(ctx, ref)
+ require.NoError(t, err, "image %s must survive a failed clone", ref)
+ defer func() { require.NoError(t, r.Close()) }()
+ data, err := io.ReadAll(r)
+ require.NoError(t, err)
+ require.NotEmpty(t, data)
+}
+
+func TestCloneRollbackRespectsResourceOwnership(t *testing.T) {
+ for _, failDuringCopy := range []bool{false, true} {
+ t.Run(fmt.Sprintf("image copy failure=%t", failDuringCopy), func(t *testing.T) {
+ t.Setenv("RETRIEVE_DRIVER", "postgres")
+ f := transferFixture(t, access.KBTransferClone)
+ catalog, physical := cloneRollbackStorage(t, f)
+ shared, err := f.svc.fileSvc.SaveBytes(f.ctx, []byte("shared image"), 7, "shared.png", false)
+ require.NoError(t, err)
+ require.NoError(t, catalog.Bind(
+ f.ctx, shared, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationExtractedImage))
+ legacy, err := physical.SaveBytes(f.ctx, []byte("legacy image"), 7, "legacy.png", false)
+ require.NoError(t, err)
+ images := []types.ImageInfo{{URL: shared}, {URL: legacy}}
+ if failDuringCopy {
+ images = append(images, types.ImageInfo{URL: "local://7/exports/missing.png"})
+ }
+ require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").
+ Update("image_info", mustImageInfoJSON(t, images)).Error)
+ f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: &partialCloneEngine{vectors: map[string]int{}}}
+ f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}}
+ f.kbs.values["other"].EmbeddingModelID = "model"
+ src, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc")
+ require.NoError(t, err)
+ err = f.svc.cloneKnowledge(f.ctx, src, f.kbs.values["other"])
+ if failDuringCopy {
+ require.ErrorContains(t, err, "failed to copy chunk image")
+ } else {
+ require.ErrorContains(t, err, "partial vector copy")
+ }
+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, shared)
+ requireCloneImageReadable(f.ctx, t, physical, legacy)
+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, "doc")
+ require.NoError(t, err)
+ require.Equal(t, []string{shared}, refs)
+ rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other")
+ require.NoError(t, err)
+ for _, dst := range rows {
+ if dst.ID == "other-doc" {
+ continue
+ }
+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, dst.ID)
+ require.NoError(t, err)
+ require.Empty(t, refs, "failed destination must release its claims")
+ }
+ // The legacy image needed a new catalog resource. Only the shared
+ // source should remain eligible for content-hash reuse after rollback.
+ ref, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("legacy image")))
+ require.NoError(t, err)
+ require.False(t, reused, "new destination-only resource must be reclaimed: %s", ref)
+ })
+ }
+}
+
+type cloneCreateFailureRepo struct{ interfaces.KnowledgeRepository }
+
+func (cloneCreateFailureRepo) CreateKnowledge(context.Context, *types.Knowledge) error {
+ return errors.New("create clone failed")
+}
+
+func TestCloneRollbackPreservesImageSourceFile(t *testing.T) {
+ for _, failedCreate := range []bool{false, true} {
+ t.Run(fmt.Sprintf("create failure=%t", failedCreate), func(t *testing.T) {
+ f := transferFixture(t, access.KBTransferClone)
+ catalog, _ := cloneRollbackStorage(t, f)
+ ref, err := f.svc.fileSvc.SaveBytes(f.ctx, []byte("image source file"), 7, "source.png", false)
+ require.NoError(t, err)
+ require.NoError(t, catalog.Bind(
+ f.ctx, ref, types.ResourceOwnerKnowledge, "doc", types.ResourceRelationSourceFile))
+ require.NoError(t, f.db.Model(&types.Knowledge{}).Where("id = ?", "doc").Update("file_path", ref).Error)
+ images := []types.ImageInfo{{URL: ref}, {URL: "local://7/exports/missing.png"}}
+ require.NoError(t, f.db.Model(&types.Chunk{}).Where("id = ?", "chunk").
+ Update("image_info", mustImageInfoJSON(t, images)).Error)
+ if failedCreate {
+ f.svc.repo = cloneCreateFailureRepo{f.repo}
+ }
+ src, err := f.repo.GetKnowledgeByID(f.ctx, 7, "doc")
+ require.NoError(t, err)
+ require.Error(t, f.svc.cloneKnowledge(f.ctx, src, f.kbs.values["other"]))
+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, ref)
+ rows, err := f.repo.ListKnowledgeByKnowledgeBaseID(f.ctx, 7, "other")
+ require.NoError(t, err)
+ for _, dst := range rows {
+ if dst.ID == "other-doc" {
+ continue
+ }
+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, dst.ID)
+ require.NoError(t, err)
+ require.Equal(t, []string{ref}, refs, "persisted destination still owns its source file")
+ }
+ })
+ }
+}
+
+// The real repositories and storage run normally; only the selected CreateChunks
+// acknowledgement is injected, so the second FAQ batch has deterministic failure.
+type faqCloneRollbackChunks struct {
+ interfaces.ChunkRepository
+ ids []string
+ creates int
+ failCreate int
+ commit bool
+ failRead bool
+}
+
+func (r *faqCloneRollbackChunks) ListAllChunksByKnowledgeID(
+ ctx context.Context, tenant uint64, id string,
+) ([]*types.Chunk, error) {
+ if r.failRead && r.creates > 0 {
+ return nil, errors.New("read saved FAQ chunks failed")
+ }
+ return r.ChunkRepository.ListAllChunksByKnowledgeID(ctx, tenant, id)
+}
+
+type cloneRollbackCatalogFault struct {
+ interfaces.ResourceCatalog
+ failList bool
+}
+
+func (c cloneRollbackCatalogFault) ListReferencesByOwner(
+ ctx context.Context, owner string, ids ...string,
+) ([]string, error) {
+ if c.failList {
+ return nil, errors.New("read resource claims failed")
+ }
+ return c.ResourceCatalog.ListReferencesByOwner(ctx, owner, ids...)
+}
+
+func (cloneRollbackCatalogFault) Release(context.Context, string, string, string) (int64, error) {
+ return 0, errors.New("release resource failed")
+}
+
+func (r *faqCloneRollbackChunks) FAQChunkDiff(
+ context.Context, uint64, string, uint64, string,
+) (*types.FAQChunkDiffResult, error) {
+ return &types.FAQChunkDiffResult{ChunksToAdd: r.ids}, nil
+}
+
+func (r *faqCloneRollbackChunks) CreateChunks(ctx context.Context, chunks []*types.Chunk) error {
+ r.creates++
+ if r.creates == r.failCreate {
+ if r.commit {
+ if err := r.ChunkRepository.CreateChunks(ctx, chunks); err != nil {
+ return err
+ }
+ }
+ return errors.New("FAQ batch write failed")
+ }
+ return r.ChunkRepository.CreateChunks(ctx, chunks)
+}
+
+func TestFAQCloneRollbackKeepsSavedImagesAndReclaimsPartialCopies(t *testing.T) {
+ for _, scenario := range []string{
+ "partial image copy", "saved batch", "existing destination", "lost acknowledgement",
+ "uncertain saved rows", "release fails", "owner lookup fails",
+ } {
+ t.Run(scenario, func(t *testing.T) {
+ t.Setenv("RETRIEVE_DRIVER", "postgres")
+ f := transferFixture(t, access.KBTransferClone)
+ catalog, physical := cloneRollbackStorage(t, f)
+ for _, kb := range f.kbs.values {
+ kb.Type = types.KnowledgeBaseTypeFAQ
+ }
+ require.NoError(t, f.db.Model(&types.Knowledge{}).Where("tenant_id = ?", 7).
+ Update("type", types.KnowledgeTypeFAQ).Error)
+ require.NoError(t, f.db.Where("tenant_id = ?", 7).Delete(&types.Chunk{}).Error)
+ f.svc.retrieveEngine = parentChildRetrieveRegistry{engine: &cloneCopyEngine{}}
+ f.svc.modelService = parentChildModelService{embedder: parentChildEmbedder{}}
+ first, err := physical.SaveBytes(f.ctx, []byte("saved image"), 7, "first.png", false)
+ require.NoError(t, err)
+ alias, err := physical.SaveBytes(f.ctx, []byte("saved image"), 7, "alias.png", false)
+ require.NoError(t, err)
+ exclusive, err := physical.SaveBytes(f.ctx, []byte("pending image"), 7, "pending.png", false)
+ require.NoError(t, err)
+ var saved string
+ if scenario == "existing destination" {
+ saved, err = f.svc.fileSvc.SaveBytes(f.ctx, []byte("saved image"), 7, "existing.png", false)
+ require.NoError(t, err)
+ require.NoError(t, catalog.Bind(
+ f.ctx, saved, types.ResourceOwnerKnowledge, "other-doc", types.ResourceRelationAttachment))
+ }
+ fault := &faqCloneRollbackChunks{
+ ChunkRepository: f.chunkRepo, failCreate: 1, commit: scenario == "lost acknowledgement",
+ failRead: scenario == "uncertain saved rows",
+ }
+ if scenario == "release fails" || scenario == "owner lookup fails" {
+ f.svc.resourceCatalog = cloneRollbackCatalogFault{catalog, scenario == "owner lookup fails"}
+ }
+ if scenario == "saved batch" {
+ fault.failCreate = 2
+ for i := 0; i < 50; i++ {
+ id := fmt.Sprintf("saved-%02d", i)
+ fault.ids = append(fault.ids, id)
+ require.NoError(t, f.db.Create(&types.Chunk{
+ ID: id, TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ,
+ ImageInfo: mustImageInfoJSON(t, []types.ImageInfo{{URL: first}}),
+ Metadata: types.JSON(`{"standard_question":"question","answers":["answer"]}`),
+ }).Error)
+ }
+ }
+ images := []types.ImageInfo{{URL: alias}, {URL: exclusive}}
+ if scenario == "partial image copy" {
+ images = append(images, types.ImageInfo{URL: "local://7/exports/missing.png"})
+ }
+ fault.ids = append(fault.ids, "pending")
+ require.NoError(t, f.db.Create(&types.Chunk{
+ ID: "pending", TenantID: 7, KnowledgeID: "doc", KnowledgeBaseID: "kb", ChunkType: types.ChunkTypeFAQ,
+ ImageInfo: mustImageInfoJSON(t, images),
+ Metadata: types.JSON(`{"standard_question":"question","answers":["answer"]}`),
+ }).Error)
+ f.svc.chunkRepo = fault
+ handledError := false
+ err = f.svc.cloneFAQKnowledgeBase(
+ f.ctx, f.kbs.values["kb"], f.kbs.values["other"], &types.KBCloneProgress{TaskID: "clone"},
+ func(*types.KBCloneProgress, error, string) { handledError = true },
+ )
+ require.Error(t, err)
+ require.True(t, handledError, "clone progress must report the failure")
+ if scenario == "owner lookup fails" {
+ require.ErrorContains(t, err, "read resource claims failed")
+ require.Zero(t, fault.creates, "do not copy or write without knowing existing claims")
+ _, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("pending image")))
+ require.NoError(t, err)
+ require.False(t, reused)
+ return
+ }
+ if scenario == "partial image copy" {
+ require.ErrorContains(t, err, "failed to copy chunk image")
+ } else {
+ require.ErrorContains(t, err, "FAQ batch write failed")
+ }
+ chunks, err := f.chunkRepo.ListAllChunksByKnowledgeID(f.ctx, 7, "other-doc")
+ require.NoError(t, err)
+ for _, chunk := range chunks {
+ for _, ref := range collectImageURLs(f.ctx, []string{chunk.ImageInfo}) {
+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, ref)
+ }
+ }
+ if saved != "" {
+ requireCloneImageReadable(f.ctx, t, f.svc.fileSvc, saved)
+ refs, err := catalog.ListReferencesByOwner(f.ctx, types.ResourceOwnerKnowledge, "other-doc")
+ require.NoError(t, err)
+ require.Equal(t, []string{saved}, refs)
+ }
+ _, reused, err := catalog.ReuseByContentHash(f.ctx, 7, cloneImageHash([]byte("pending image")))
+ require.NoError(t, err)
+ keep := scenario == "lost acknowledgement" || scenario == "uncertain saved rows" ||
+ scenario == "release fails"
+ require.Equal(t, keep, reused, "keep saved images and fail closed when ownership is uncertain")
+ })
+ }
+}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 7ffd22f00..c0e697714 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -114,7 +114,10 @@ func (s *knowledgeService) cloneKnowledge(
}
stored, loadErr := s.repo.GetKnowledgeByID(ctx, dst.TenantID, dst.ID)
if errors.Is(loadErr, repository.ErrKnowledgeNotFound) {
- cleanupCopiedObjects(ctx, s.resolveFileService(ctx, targetKB), copiedFilePaths)
+ cleanupCopiedObjects(
+ ctx, s.resolveFileService(ctx, targetKB),
+ knowledgeResourceOwners(s.resourceCatalog, dst.ID), copiedFilePaths,
+ )
return
}
// An uncertain save/read must not delete objects that may be referenced.