SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: self-heal chunk manifest reads when volume locations go stale (#11107)
Chris Lu0f05957bc4d8
diff --git a/weed/filer/filechunk_group.go b/weed/filer/filechunk_group.goindex 87c7008479..0052ba707e 100644--- a/weed/filer/filechunk_group.go+++ b/weed/filer/filechunk_group.go@@ -18,6 +18,8 @@ type ChunkGroup struct {sectionsLock sync.RWMutexreaderCache *ReaderCacheconcurrentReaders int+ // cacheInvalidator lets manifest resolution drop stale volume locations, as ReaderCache does for chunk reads+ cacheInvalidator CacheInvalidator}// NewChunkGroup creates a ChunkGroup with configurable concurrency.@@ -43,6 +45,7 @@ func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_sections: make(map[SectionIndex]*FileChunkSection),readerCache: NewReaderCache(readerCacheLimit, chunkCache, lookupFn, cacheInvalidator),concurrentReaders: concurrentReaders,+ cacheInvalidator: cacheInvalidator,}err := group.SetChunks(chunks)@@ -227,7 +230,7 @@ func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {continue}- resolvedChunks, err := ResolveOneChunkManifest(context.Background(), group.lookupFn, chunk)+ resolvedChunks, err := ResolveOneChunkManifest(context.Background(), group.lookupFn, chunk, group.cacheInvalidator)if err != nil {return err}diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.goindex f29cbf2b23..57cef8fa7f 100644--- a/weed/filer/filechunk_manifest.go+++ b/weed/filer/filechunk_manifest.go@@ -49,7 +49,7 @@ func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonMareturn}-func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {+func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64, invalidator CacheInvalidator) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {// TODO maybe parallel thisfor _, chunk := range chunks {@@ -62,14 +62,14 @@ func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFilcontinue}- resolvedChunks, err := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk)+ resolvedChunks, err := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk, invalidator)if err != nil {return dataChunks, nil, err}manifestChunks = append(manifestChunks, chunk)// recursive- subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset)+ subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset, invalidator)if subErr != nil {return dataChunks, nil, subErr}@@ -79,7 +79,7 @@ func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFilreturn}-func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {+func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk, invalidator CacheInvalidator) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {if !chunk.IsChunkManifest {return}@@ -88,13 +88,13 @@ func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupbytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)bytesBuffer.Reset()defer bytesBufferPool.Put(bytesBuffer)- err := fetchWholeChunk(ctx, bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)+ err := fetchWholeChunk(ctx, bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed, invalidator)if err != nil {- return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)+ return nil, fmt.Errorf("fail to read manifest %s: %w", chunk.GetFileIdString(), err)}m := &filer_pb.FileChunkManifest{}if err := proto.Unmarshal(bytesBuffer.Bytes(), m); err != nil {- return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err)+ return nil, fmt.Errorf("fail to unmarshal manifest %s: %w", chunk.GetFileIdString(), err)}// recursive@@ -103,18 +103,27 @@ func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.Lookup}// TODO fetch from cache for weed mount?-func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error {+func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, invalidator CacheInvalidator) error {urlStrings, err := lookupFileIdFn(ctx, fileId)if err != nil {glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)return err}jwt := JwtForVolumeServer(fileId)- _, err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, jwt, cipherKey, isGzipped, true, 0, 0)- if err != nil {- return err+ if _, err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, jwt, cipherKey, isGzipped, true, 0, 0); err == nil {+ return nil}- return nil+ if ctxErr := ctx.Err(); ctxErr != nil {+ // a cancelled read says nothing about where the volume lives, and the+ // stream error it provoked is a symptom, not the cause+ return ctxErr+ }+ return retryFetchWithFreshLocations(ctx, invalidator, lookupFileIdFn, fileId, urlStrings, err, func(newUrls []string) error {+ // the failed attempt may have streamed a partial prefix into the buffer+ bytesBuffer.Reset()+ _, retryErr := retriedStreamFetchChunkData(ctx, bytesBuffer, newUrls, jwt, cipherKey, isGzipped, true, 0, 0)+ return retryErr+ })}func fetchChunkRange(ctx context.Context, buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, refreshUrls util_http.RefreshUrlsFunc) (int, error) {diff --git a/weed/filer/filechunk_manifest_test.go b/weed/filer/filechunk_manifest_test.goindex 4411b01fba..e9243be690 100644--- a/weed/filer/filechunk_manifest_test.go+++ b/weed/filer/filechunk_manifest_test.go@@ -6,6 +6,10 @@ import ("fmt""io""math"+ "net/http"+ "net/http/httptest"+ "strconv"+ "sync/atomic""testing""github.com/stretchr/testify/assert"@@ -504,3 +508,155 @@ func TestManifestBloatDetection(t *testing.T) {}}}++// countingInvalidator stands in for the vidMap cache a mount or filer hands to+// manifest resolution, recording how often the locations were dropped.+type countingInvalidator struct {+ invalidations atomic.Int32+}++func (inv *countingInvalidator) InvalidateCache(fileId string) {+ inv.invalidations.Add(1)+}++// stagedLookup returns staleUrls once and freshUrls afterwards, the way a+// vidMapClient re-reads the master once InvalidateCache has dropped the entry.+type stagedLookup struct {+ staleUrls []string+ freshUrls []string+ calls atomic.Int32+}++func (l *stagedLookup) lookup(ctx context.Context, fileId string) ([]string, error) {+ if l.calls.Add(1) == 1 {+ return l.staleUrls, nil+ }+ return l.freshUrls, nil+}++func manifestServer(t *testing.T, manifestBytes []byte) *httptest.Server {+ t.Helper()+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)))+ w.Write(manifestBytes)+ }))+ t.Cleanup(srv.Close)+ return srv+}++func fetchManifestBuffer(t *testing.T) *bytes.Buffer {+ t.Helper()+ bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)+ bytesBuffer.Reset()+ t.Cleanup(func() { bytesBufferPool.Put(bytesBuffer) })+ return bytesBuffer+}++// TestFetchWholeChunkRetriesFreshLocations covers the mount reading a multipart+// file whose manifest volume has moved: the cached location is dead, so the+// fetch has to drop it and come back with what the master knows now. The stale+// server streams a prefix before dying, which the retry must not keep.+func TestFetchWholeChunkRetriesFreshLocations(t *testing.T) {+ manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{+ Chunks: []*filer_pb.FileChunk{+ {FileId: "100,abc", Offset: 0, Size: 8},+ {FileId: "101,def", Offset: 8, Size: 8},+ },+ })+ assert.NoError(t, err)++ staleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)+64))+ w.Write([]byte("garbage-from-stale-server"))+ w.(http.Flusher).Flush()+ panic(http.ErrAbortHandler)+ }))+ defer staleSrv.Close()++ lookup := &stagedLookup{+ staleUrls: []string{staleSrv.URL + "/5,stale"},+ freshUrls: []string{manifestServer(t, manifestBytes).URL + "/5,stale"},+ }+ inv := &countingInvalidator{}+ bytesBuffer := fetchManifestBuffer(t)++ assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,stale", nil, false, inv))+ assert.Equal(t, int32(1), inv.invalidations.Load())+ assert.Equal(t, int32(2), lookup.calls.Load())++ // the buffer must hold the fresh manifest alone, not the stale prefix ahead of it+ decoded := &filer_pb.FileChunkManifest{}+ assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded))+ assertEqualChunks(t, []*filer_pb.FileChunk{+ {FileId: "100,abc", Offset: 0, Size: 8},+ {FileId: "101,def", Offset: 8, Size: 8},+ }, decoded.Chunks)+}++// TestFetchWholeChunkWithoutInvalidator keeps the old behavior for callers whose+// lookup function has no cache to drop: the fetch error surfaces as it is.+func TestFetchWholeChunkWithoutInvalidator(t *testing.T) {+ failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ defer failSrv.Close()++ lookup := &stagedLookup{+ staleUrls: []string{failSrv.URL + "/5,abc"},+ freshUrls: []string{"http://unused:8080/5,abc"},+ }++ assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, nil))+ assert.Equal(t, int32(1), lookup.calls.Load())+}++// TestFetchWholeChunkUnchangedLocations guards against looping: when the master+// still reports the servers that just failed, there is nothing new to try.+func TestFetchWholeChunkUnchangedLocations(t *testing.T) {+ failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ defer failSrv.Close()++ lookup := &stagedLookup{+ staleUrls: []string{failSrv.URL + "/5,abc"},+ freshUrls: []string{failSrv.URL + "/5,abc"},+ }+ inv := &countingInvalidator{}++ assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv))+ assert.Equal(t, int32(2), lookup.calls.Load())+ assert.Equal(t, int32(1), inv.invalidations.Load())+}++// TestFetchWholeChunkCancelledKeepsLocations checks that a caller walking away+// mid-read does not cost every other reader a master round trip.+func TestFetchWholeChunkCancelledKeepsLocations(t *testing.T) {+ lookup := &stagedLookup{+ staleUrls: []string{"http://unused:8080/5,abc"},+ freshUrls: []string{"http://elsewhere:8080/5,abc"},+ }+ inv := &countingInvalidator{}++ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ err := fetchWholeChunk(ctx, fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv)+ assert.ErrorIs(t, err, context.Canceled)+ assert.Equal(t, int32(0), inv.invalidations.Load())+ assert.Equal(t, int32(1), lookup.calls.Load())++ // the cancellation must survive the wrapping ResolveOneChunkManifest does,+ // or callers cannot tell a dead volume server from their own abort+ manifestChunk := &filer_pb.FileChunk{FileId: "5,abc", IsChunkManifest: true}+ _, resolveErr := ResolveOneChunkManifest(ctx, lookup.lookup, manifestChunk, inv)+ assert.ErrorIs(t, resolveErr, context.Canceled)++ // volume.fsck resolves manifests with no invalidator and still has to tell+ // its own abort from a corrupt manifest, so the nil path keeps the identity+ noInvalidator := retryFetchWithFreshLocations(ctx, nil, lookup.lookup, "5,abc", nil, fmt.Errorf("stale server said no"), func([]string) error {+ t.Fatal("refetch must not run on a cancelled read")+ return nil+ })+ assert.ErrorIs(t, noInvalidator, context.Canceled)+}diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.goindex 6e9ec3f218..8cd8bb8436 100644--- a/weed/filer/filechunks.go+++ b/weed/filer/filechunks.go@@ -100,13 +100,13 @@ func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, streturn}-func MinusChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) {+func MinusChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk, invalidator CacheInvalidator) (delta []*filer_pb.FileChunk, err error) {- aData, aMeta, aErr := ResolveChunkManifest(ctx, lookupFileIdFn, as, 0, math.MaxInt64)+ aData, aMeta, aErr := ResolveChunkManifest(ctx, lookupFileIdFn, as, 0, math.MaxInt64, invalidator)if aErr != nil {return nil, aErr}- bData, bMeta, bErr := ResolveChunkManifest(ctx, lookupFileIdFn, bs, 0, math.MaxInt64)+ bData, bMeta, bErr := ResolveChunkManifest(ctx, lookupFileIdFn, bs, 0, math.MaxInt64, invalidator)if bErr != nil {return nil, bErr}@@ -271,7 +271,7 @@ func MergeIntoChunkViews(chunkViews *IntervalList[*ChunkView], start int64, stop// If the file chunk content is a chunk manifestfunc NonOverlappingVisibleIntervals(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval], err error) {- chunks, _, err = ResolveChunkManifest(ctx, lookupFileIdFn, chunks, startOffset, stopOffset)+ chunks, _, err = ResolveChunkManifest(ctx, lookupFileIdFn, chunks, startOffset, stopOffset, nil)if err != nil {return}diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.goindex 63a20251da..f32b97a8bb 100644--- a/weed/filer/filer_deletion.go+++ b/weed/filer/filer_deletion.go@@ -602,7 +602,7 @@ func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunkf.FileIdDeletionQueue.EnQueue(chunk.GetFileIdString())continue}- dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk)+ dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk, f.MasterClient)if manifestResolveErr != nil {glog.V(0).InfofCtx(ctx, "failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr)}@@ -628,7 +628,7 @@ func (f *Filer) deleteChunksIfNotNew(ctx context.Context, oldEntry, newEntry *EnnewChunks = newEntry.GetChunks()}- toDelete, err := MinusChunks(ctx, f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks)+ toDelete, err := MinusChunks(ctx, f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks, f.MasterClient)if err != nil {glog.ErrorfCtx(ctx, "Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s", newChunks, oldChunks)returndiff --git a/weed/filer/persisted_log_cache.go b/weed/filer/persisted_log_cache.goindex 6f33a8c444..430582c21c 100644--- a/weed/filer/persisted_log_cache.go+++ b/weed/filer/persisted_log_cache.go@@ -190,7 +190,7 @@ func loadLogFileEntries(masterClient *wdclient.MasterClient, chunk *filer_pb.FillookupFileIdFn := func(ctx context.Context, fileId string) (targetUrls []string, err error) {return masterClient.LookupFileId(ctx, fileId)}- if fetchErr := fetchWholeChunk(context.Background(), bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed); fetchErr != nil {+ if fetchErr := fetchWholeChunk(context.Background(), bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed, masterClient); fetchErr != nil {return nil, false, fetchErr}return decodeLogRecords(bytesBuffer.Bytes())diff --git a/weed/filer/stream.go b/weed/filer/stream.goindex b06e4a55ae..5b4d4cbd5b 100644--- a/weed/filer/stream.go+++ b/weed/filer/stream.go@@ -107,6 +107,12 @@ type VolumeServerJwtFunction func(fileId string) string// resolved locations actually changed (so we never retry against the same servers). originalErr// is returned unchanged when no retry is attempted, so callers surface the real fetch failure.func retryFetchWithFreshLocations(ctx context.Context, invalidator CacheInvalidator, lookupFn wdclient.LookupFileIdFunctionType, fileId string, oldUrls []string, originalErr error, refetch func(newUrls []string) error) error {+ // the caller may have gone away between its own check and this one; a+ // cancelled read is no evidence the locations are wrong, and callers such+ // as volume.fsck tell an abort from real corruption with errors.Is+ if ctxErr := ctx.Err(); ctxErr != nil {+ return ctxErr+ }if invalidator == nil {return originalErr}diff --git a/weed/mount/peer_fetcher.go b/weed/mount/peer_fetcher.goindex 87d816c1fd..ad95e42fa1 100644--- a/weed/mount/peer_fetcher.go+++ b/weed/mount/peer_fetcher.go@@ -65,7 +65,7 @@ func (fh *FileHandle) tryPeerRead(ctx context.Context, fileSize int64, buff []byif readStop > fileSize {readStop = fileSize}- dataChunks, _, err := filer.ResolveChunkManifest(ctx, fh.wfs.LookupFn(), chunks, offset, readStop)+ dataChunks, _, err := filer.ResolveChunkManifest(ctx, fh.wfs.LookupFn(), chunks, offset, readStop, fh.wfs.CacheInvalidator())if err != nil {return 0, 0, fmt.Errorf("resolve manifest: %w", err)}diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.goindex b05a926b61..08a79bffcb 100644--- a/weed/replication/sink/filersink/fetch_write.go+++ b/weed/replication/sink/filersink/fetch_write.go@@ -195,7 +195,7 @@ func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunkresolveName := fmt.Sprintf("resolve manifest %s", sourceChunk.GetFileIdString())missingGate := fs.newMissingSourceChunkGate(sourceChunk.GetFileIdString())err := util.RetryUntil(resolveName, func() error {- rc, e := filer.ResolveOneChunkManifest(ctx, fs.filerSource.LookupFileId, sourceChunk)+ rc, e := filer.ResolveOneChunkManifest(ctx, fs.filerSource.LookupFileId, sourceChunk, nil)if e != nil {return e}diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.goindex 4552312be3..0f7a42e10b 100644--- a/weed/replication/sink/filersink/filer_sink.go+++ b/weed/replication/sink/filersink/filer_sink.go@@ -405,11 +405,11 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent}func compareChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) {- aData, aMeta, aErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, oldEntry.GetChunks(), 0, math.MaxInt64)+ aData, aMeta, aErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, oldEntry.GetChunks(), 0, math.MaxInt64, nil)if aErr != nil {return nil, nil, aErr}- bData, bMeta, bErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, newEntry.GetChunks(), 0, math.MaxInt64)+ bData, bMeta, bErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, newEntry.GetChunks(), 0, math.MaxInt64, nil)if bErr != nil {return nil, nil, bErr}diff --git a/weed/s3api/s3api_chunk_manifest.go b/weed/s3api/s3api_chunk_manifest.goindex c25a36ca23..d8eaaa9e83 100644--- a/weed/s3api/s3api_chunk_manifest.go+++ b/weed/s3api/s3api_chunk_manifest.go@@ -65,7 +65,7 @@ func (s3a *S3ApiServer) flattenManifestChunks(ctx context.Context, entry *filer_if entry == nil || !filer.HasChunkManifest(entry.GetChunks()) {return nil, nil}- dataChunks, manifestChunks, err := filer.ResolveChunkManifest(ctx, s3a.createLookupFileIdFunction(), entry.GetChunks(), 0, math.MaxInt64)+ dataChunks, manifestChunks, err := filer.ResolveChunkManifest(ctx, s3a.createLookupFileIdFunction(), entry.GetChunks(), 0, math.MaxInt64, s3a.filerClient)if err != nil {return nil, err}diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.goindex 99b80d6528..902d983a8b 100644--- a/weed/s3api/s3api_object_handlers.go+++ b/weed/s3api/s3api_object_handlers.go@@ -2140,7 +2140,7 @@ func (s3a *S3ApiServer) getEncryptedStreamFromVolumes(ctx context.Context, entrylookupFileIdFn := s3a.createLookupFileIdFunction()// Resolve chunks- resolvedChunks, _, err := filer.ResolveChunkManifest(ctx, lookupFileIdFn, chunks, offset, offset+size)+ resolvedChunks, _, err := filer.ResolveChunkManifest(ctx, lookupFileIdFn, chunks, offset, offset+size, s3a.filerClient)if err != nil {return nil, err}diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.goindex 0168a9279c..c2d21922c9 100644--- a/weed/server/filer_grpc_server.go+++ b/weed/server/filer_grpc_server.go@@ -715,7 +715,7 @@ func (fs *FilerServer) cleanupChunks(ctx context.Context, fullpath string, exist// remove old chunks if not included in the new onesif existingEntry != nil {- garbage, err = filer.MinusChunks(ctx, fs.lookupFileId, existingEntry.GetChunks(), newEntry.GetChunks())+ garbage, err = filer.MinusChunks(ctx, fs.lookupFileId, existingEntry.GetChunks(), newEntry.GetChunks(), fs.filer.MasterClient)if err != nil {return newEntry.GetChunks(), nil, fmt.Errorf("MinusChunks: %w", err)}diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.goindex d44bad2832..020a59c71a 100644--- a/weed/server/filer_grpc_server_rename.go+++ b/weed/server/filer_grpc_server_rename.go@@ -274,7 +274,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seawee}}if existingTarget != nil {- toDelete, err := filer.MinusChunks(ctx, fs.filer.MasterClient.GetLookupFileIdFunction(), existingTarget.GetChunks(), newEntry.GetChunks())+ toDelete, err := filer.MinusChunks(ctx, fs.filer.MasterClient.GetLookupFileIdFunction(), existingTarget.GetChunks(), newEntry.GetChunks(), fs.filer.MasterClient)if err != nil {glog.ErrorfCtx(ctx, "Failed to resolve overwrite target chunks during rename. new: %v, old: %v", newEntry.GetChunks(), existingTarget.GetChunks())} else if len(toDelete) > 0 {diff --git a/weed/server/filer_server_handlers_copy.go b/weed/server/filer_server_handlers_copy.goindex 2a6364b763..9db334b1de 100644--- a/weed/server/filer_server_handlers_copy.go+++ b/weed/server/filer_server_handlers_copy.go@@ -600,7 +600,7 @@ func (fs *FilerServer) copyChunksWithManifest(ctx context.Context, srcChunks []*return fs.filer.MasterClient.GetLookupFileIdFunction()(ctx, fileId)}- resolvedChunks, err := filer.ResolveOneChunkManifest(ctx, lookupFileIdFn, manifestChunk)+ resolvedChunks, err := filer.ResolveOneChunkManifest(ctx, lookupFileIdFn, manifestChunk, fs.filer.MasterClient)if err != nil {return nil, fmt.Errorf("failed to resolve manifest chunk %s: %w", manifestChunk.GetFileIdString(), err)}diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.goindex 2992f31324..8de704aaa3 100644--- a/weed/server/filer_server_handlers_read.go+++ b/weed/server/filer_server_handlers_read.go@@ -132,7 +132,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)if entry.Chunks, _, err = filer.ResolveChunkManifest(ctx,fs.filer.MasterClient.GetLookupFileIdFunction(),- entry.GetChunks(), 0, math.MaxInt64); err != nil {+ entry.GetChunks(), 0, math.MaxInt64, fs.filer.MasterClient); err != nil {err = fmt.Errorf("failed to resolve chunk manifest, err: %s", err.Error())writeJsonError(w, r, http.StatusInternalServerError, err)returndiff --git a/weed/server/filer_server_handlers_write_merge.go b/weed/server/filer_server_handlers_write_merge.goindex 24e642bd60..157d96c797 100644--- a/weed/server/filer_server_handlers_write_merge.go+++ b/weed/server/filer_server_handlers_write_merge.go@@ -64,7 +64,7 @@ func (fs *FilerServer) mergeChunks(ctx context.Context, so *operation.StorageOpt}}- garbage, err := filer.MinusChunks(ctx, fs.lookupFileId, inputChunks, mergedChunks)+ garbage, err := filer.MinusChunks(ctx, fs.lookupFileId, inputChunks, mergedChunks, fs.filer.MasterClient)if err != nil {glog.ErrorfCtx(ctx, "Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s",mergedChunks, inputChunks)diff --git a/weed/shell/command_fs_distribute_chunks.go b/weed/shell/command_fs_distribute_chunks.goindex b66df41dc2..96baaa7403 100644--- a/weed/shell/command_fs_distribute_chunks.go+++ b/weed/shell/command_fs_distribute_chunks.go@@ -255,7 +255,7 @@ func resolveEntryDataChunks(commandEnv *CommandEnv, entry *filer_pb.Entry) (dataif !filer.HasChunkManifest(chunks) {return chunks, false, nil}- dataChunks, _, err = filer.ResolveChunkManifest(context.Background(), filer.LookupFn(commandEnv), chunks, 0, math.MaxInt64)+ dataChunks, _, err = filer.ResolveChunkManifest(context.Background(), filer.LookupFn(commandEnv), chunks, 0, math.MaxInt64, nil)if err != nil {return nil, true, fmt.Errorf("resolve chunk manifest: %v", err)}diff --git a/weed/shell/command_fs_merge_volumes.go b/weed/shell/command_fs_merge_volumes.goindex 393cfd84b7..4880067719 100644--- a/weed/shell/command_fs_merge_volumes.go+++ b/weed/shell/command_fs_merge_volumes.go@@ -676,7 +676,7 @@ func (c *commandFsMergeVolumes) rewriteManifestChunk(return chunk, false, rewrittenNeedles{}, fmt.Errorf("not a manifest chunk: %s", chunk.GetFileIdString())}- subChunks, err := filer.ResolveOneChunkManifest(ctx, lookupFn, chunk)+ subChunks, err := filer.ResolveOneChunkManifest(ctx, lookupFn, chunk, nil)if err != nil {return chunk, false, rewrittenNeedles{}, err}diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.goindex a96fc09d1c..92aed41698 100644--- a/weed/shell/command_fs_verify.go+++ b/weed/shell/command_fs_verify.go@@ -287,7 +287,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errCreturn nil}}- dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64)+ dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64, nil)if resolveErr != nil {return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr)}diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.goindex 76d344f826..22d96b7b3f 100644--- a/weed/shell/command_volume_fsck.go+++ b/weed/shell/command_volume_fsck.go@@ -310,7 +310,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo mif *c.verbose && entry.Entry.IsDirectory {fmt.Fprintf(c.writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))}- dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(ctx, filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64)+ dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(ctx, filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64, nil)if resolveErr != nil {// Cancellation/deadline isn't manifest corruption; surface it// so the BFS bails out cleanly without polluting the