SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
[Filer] Parallelize Chunk Manifest Resolution to Reduce Large File Read Latency (#11215)
ssshr-66361fd6b263d8
diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.goindex 35fdc5063a..597861a87a 100644--- a/weed/filer/filechunk_manifest.go+++ b/weed/filer/filechunk_manifest.go@@ -3,6 +3,7 @@ package filerimport ("bytes""context"+ "errors""fmt""io""math"@@ -25,6 +26,14 @@ var bytesBufferPool = sync.Pool{},}+// Keep Manifest reads bounded across all recursion levels of one resolution.+const maxChunkManifestResolveWorkers = 4++// Size of the job queue buffer. Large enough that submission does not block+// under normal chunk counts, so a promptly-failing manifest is always queued+// and can cancel stalled sibling reads once a worker picks it up.+const chunkManifestResolveJobBufferSize = 128+func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {for _, chunk := range chunks {if chunk.IsChunkManifest {@@ -46,26 +55,229 @@ func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonMa}func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64, invalidator CacheInvalidator) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {- // TODO maybe parallel this- for _, chunk := range chunks {+ resolver := newChunkManifestResolver(ctx, lookupFileIdFn, invalidator)+ defer resolver.close()+ return resolver.resolve(chunks, startOffset, stopOffset)+}++type chunkManifestResolveJob struct {+ chunk *filer_pb.FileChunk+ result *chunkManifestResolveResult+ done *sync.WaitGroup+ batchCtx context.Context+ batchCancel context.CancelFunc+ batchOnce *sync.Once+}++type chunkManifestResolveResult struct {+ chunks []*filer_pb.FileChunk+ err error+ internalCancel bool+}++type chunkManifestResolver struct {+ ctx context.Context+ parentCtx context.Context+ cancel context.CancelFunc+ lookupFileIdFn wdclient.LookupFileIdFunctionType+ invalidator CacheInvalidator+ jobs chan chunkManifestResolveJob+ overflowSem chan struct{}+ workers sync.WaitGroup+ startOnce sync.Once+ started bool+}++func newChunkManifestResolver(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, invalidator CacheInvalidator) *chunkManifestResolver {+ workCtx, cancel := context.WithCancel(ctx)+ resolver := &chunkManifestResolver{+ ctx: workCtx,+ parentCtx: ctx,+ cancel: cancel,+ lookupFileIdFn: lookupFileIdFn,+ invalidator: invalidator,+ jobs: make(chan chunkManifestResolveJob, chunkManifestResolveJobBufferSize),+ overflowSem: make(chan struct{}, maxChunkManifestResolveWorkers),+ }+ return resolver+}++func (r *chunkManifestResolver) executeJob(job chunkManifestResolveJob) {+ job.result.chunks, job.result.err = ResolveOneChunkManifest(job.batchCtx, r.lookupFileIdFn, job.chunk, r.invalidator)+ if job.result.err != nil && r.parentCtx.Err() == nil {+ if job.batchCtx.Err() != nil && errors.Is(job.result.err, context.Canceled) {+ job.result.internalCancel = true+ } else if job.batchCtx.Err() == nil {+ job.batchOnce.Do(job.batchCancel)+ }+ }+ job.done.Done()+}++func (r *chunkManifestResolver) executeOverflowJob(job chunkManifestResolveJob) {+ select {+ case r.overflowSem <- struct{}{}:+ defer func() { <-r.overflowSem }()+ r.executeJob(job)+ case <-job.batchCtx.Done():+ if job.result.err == nil {+ job.result.err = job.batchCtx.Err()+ job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil+ }+ job.done.Done()+ case <-r.ctx.Done():+ if job.result.err == nil {+ job.result.err = r.ctx.Err()+ job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil+ }+ job.done.Done()+ }+}++func (r *chunkManifestResolver) worker() {+ defer r.workers.Done()+ for job := range r.jobs {+ r.executeJob(job)+ }+}++func (r *chunkManifestResolver) close() {+ r.cancel()+ close(r.jobs)+ if r.started {+ r.workers.Wait()+ }+}++func (r *chunkManifestResolver) submit(job chunkManifestResolveJob) bool {+ r.startOnce.Do(func() {+ r.workers.Add(maxChunkManifestResolveWorkers)+ for i := 0; i < maxChunkManifestResolveWorkers; i++ {+ go r.worker()+ }+ r.started = true+ })+ select {+ case r.jobs <- job:+ return true+ case <-r.ctx.Done():+ return false+ case <-job.batchCtx.Done():+ // Batch was cancelled by a sibling failure; don't queue this job.+ // Set the result so the caller doesn't overwrite it; the caller+ // owns the WaitGroup decrement.+ job.result.err = job.batchCtx.Err()+ job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil+ return false+ default:+ // Buffer is full (exceedingly rare: more than chunkManifestResolveJobBufferSize+ // in-range manifests at one level). Run the job in a bounded overflow goroutine+ // so a promptly-failing manifest can still cancel the batch without waiting for+ // a worker slot, while keeping total concurrency bounded.+ go r.executeOverflowJob(job)+ return true+ }+}+func (r *chunkManifestResolver) resolve(chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {+ type resolveSlot struct {+ chunk *filer_pb.FileChunk+ result chunkManifestResolveResult+ }++ // Cancellation is scoped to this parallel read batch so a failure in a+ // later manifest does not cancel recursive work for an earlier manifest+ // that already completed. Recursion creates its own batch context derived+ // from the still-alive resolver context.+ batchCtx, batchCancel := context.WithCancel(r.ctx)+ defer batchCancel()+ var batchOnce sync.Once++ slots := make([]resolveSlot, len(chunks))+ var reads sync.WaitGroup+ for i, chunk := range chunks {if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {continue}+ slots[i].chunk = chunkif !chunk.IsChunkManifest {- dataChunks = append(dataChunks, chunk)continue}- resolvedChunks, err := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk, invalidator)- if err != nil {- return dataChunks, nil, err+ reads.Add(1)+ if !r.submit(chunkManifestResolveJob{+ chunk: chunk,+ result: &slots[i].result,+ done: &reads,+ batchCtx: batchCtx,+ batchCancel: batchCancel,+ batchOnce: &batchOnce,+ }) {+ // submit may have already set the result (batch cancellation).+ // Only set it here for the resolver-cancellation case.+ if slots[i].result.err == nil {+ slots[i].result.err = r.ctx.Err()+ slots[i].result.internalCancel = r.parentCtx.Err() == nil && slots[i].result.err != nil+ }+ reads.Done()+ }+ }+ reads.Wait()+ if err := r.parentCtx.Err(); err != nil {+ return dataChunks, nil, err+ }++ // Recurse only after this level's reads release their worker slots. A+ // worker must never wait for a child manifest while holding a slot.+ //+ // Pre-scan for the first real (non-internal-cancel) error before+ // recursing. If a later manifest already failed, return its error+ // promptly with data chunks that are already in hand, instead of+ // blocking on recursive reads of earlier manifests' children.+ for i, slot := range slots {+ if slot.chunk == nil {+ continue+ }+ if slot.result.err != nil {+ if slot.result.internalCancel {+ continue+ }+ for j := 0; j < i; j++ {+ if slots[j].chunk == nil {+ continue+ }+ if !slots[j].chunk.IsChunkManifest {+ dataChunks = append(dataChunks, slots[j].chunk)+ continue+ }+ for _, c := range slots[j].result.chunks {+ if !c.IsChunkManifest && max(c.Offset, startOffset) < min(c.Offset+int64(c.Size), stopOffset) {+ dataChunks = append(dataChunks, c)+ }+ }+ }+ return dataChunks, nil, slot.result.err+ }+ }++ for _, slot := range slots {+ if slot.chunk == nil {+ continue+ }+ if slot.result.err != nil {+ if slot.result.internalCancel {+ continue+ }+ return dataChunks, nil, slot.result.err+ }+ if !slot.chunk.IsChunkManifest {+ dataChunks = append(dataChunks, slot.chunk)+ continue}- manifestChunks = append(manifestChunks, chunk)- // recursive- subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset, invalidator)+ manifestChunks = append(manifestChunks, slot.chunk)+ subDataChunks, subManifestChunks, subErr := r.resolve(slot.result.chunks, startOffset, stopOffset)if subErr != nil {return dataChunks, nil, subErr}diff --git a/weed/filer/filechunk_manifest_resolve_test.go b/weed/filer/filechunk_manifest_resolve_test.gonew file mode 100644index 0000000000..0daf1f2749--- /dev/null+++ b/weed/filer/filechunk_manifest_resolve_test.go@@ -0,0 +1,595 @@+package filer++import (+ "context"+ "errors"+ "fmt"+ "net"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/proto"+)++type manifestReadFixture struct {+ server *httptest.Server+ mu sync.Mutex+ started []string+ completed []string+ active atomic.Int32+ maxActive atomic.Int32+ loads atomic.Int32+ manifests map[string][]byte+ delays map[string]time.Duration+ stopReads chan struct{}+}++func newManifestReadFixture(t testing.TB, manifests map[string][]*filer_pb.FileChunk, delays map[string]time.Duration) *manifestReadFixture {+ t.Helper()+ encoded := make(map[string][]byte, len(manifests))+ for id, chunks := range manifests {+ data, err := proto.Marshal(&filer_pb.FileChunkManifest{Chunks: chunks})+ require.NoError(t, err)+ encoded[id] = data+ }++ fixture := &manifestReadFixture{manifests: encoded, delays: delays, stopReads: make(chan struct{})}+ listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")+ require.NoError(t, err)+ fixture.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ id := strings.TrimPrefix(r.URL.Path, "/")+ fixture.loads.Add(1)+ fixture.active.Add(1)+ fixture.updateMaxActive()+ fixture.mu.Lock()+ fixture.started = append(fixture.started, id)+ fixture.mu.Unlock()+ finishRead := func() {+ fixture.active.Add(-1)+ fixture.mu.Lock()+ fixture.completed = append(fixture.completed, id)+ fixture.mu.Unlock()+ }++ if delay := fixture.delays[id]; delay > 0 {+ timer := time.NewTimer(delay)+ select {+ case <-r.Context().Done():+ timer.Stop()+ finishRead()+ return+ case <-fixture.stopReads:+ timer.Stop()+ finishRead()+ return+ case <-timer.C:+ }+ }+ data, ok := fixture.manifests[id]+ if !ok {+ http.NotFound(w, r)+ finishRead()+ return+ }+ w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write(data)+ finishRead()+ }))+ fixture.server.Listener = listener+ fixture.server.Start()+ t.Cleanup(fixture.server.Close)+ return fixture+}++func (f *manifestReadFixture) updateMaxActive() {+ for {+ current := f.active.Load()+ maximum := f.maxActive.Load()+ if current <= maximum || f.maxActive.CompareAndSwap(maximum, current) {+ return+ }+ }+}++func (f *manifestReadFixture) lookup(ctx context.Context, fileID string) ([]string, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ return []string{f.server.URL + "/" + fileID}, nil+}++func (f *manifestReadFixture) completionOrder() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return append([]string(nil), f.completed...)+}++func resolveTestManifest(id string, offset int64) *filer_pb.FileChunk {+ return &filer_pb.FileChunk{FileId: id, IsChunkManifest: true, Offset: offset, Size: 100}+}++func resolveTestData(id string, offset int64) *filer_pb.FileChunk {+ return &filer_pb.FileChunk{FileId: id, Offset: offset, Size: 10}+}++func TestResolveChunkManifestParallelReadsAreBounded(t *testing.T) {+ manifests := make(map[string][]*filer_pb.FileChunk)+ inputs := make([]*filer_pb.FileChunk, 0, 8)+ delays := make(map[string]time.Duration)+ for i := 0; i < 8; i++ {+ id := fmt.Sprintf("m%d", i)+ manifests[id] = []*filer_pb.FileChunk{resolveTestData(fmt.Sprintf("d%d", i), int64(i*10))}+ inputs = append(inputs, resolveTestManifest(id, int64(i*100)))+ delays[id] = 50 * time.Millisecond+ }+ fixture := newManifestReadFixture(t, manifests, delays)++ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, inputs, 0, 1000, nil)+ require.NoError(t, err)+ require.Len(t, data, len(inputs))+ require.Len(t, meta, len(inputs))+ require.GreaterOrEqual(t, fixture.maxActive.Load(), int32(2), "independent manifests should overlap")+ require.LessOrEqual(t, fixture.maxActive.Load(), int32(4), "manifest reads must be bounded")+}++func TestResolveChunkManifestPreservesInputOrderWhenReadsCompleteOutOfOrder(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "slow": {resolveTestData("slow-data", 0)},+ "fast": {resolveTestData("fast-data", 100)},+ },+ map[string]time.Duration{"slow": 80 * time.Millisecond, "fast": 5 * time.Millisecond},+ )++ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("slow", 0),+ resolveTestManifest("fast", 100),+ }, 0, 200, nil)+ require.NoError(t, err)+ require.Equal(t, []string{"slow-data", "fast-data"}, fileIDs(data))+ require.Equal(t, []string{"slow", "fast"}, fileIDs(meta))+ require.Equal(t, []string{"fast", "slow"}, fixture.completionOrder())+}++func TestResolveChunkManifestNestedManifestsKeepDepthFirstOrder(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "parent": {resolveTestData("parent-data", 0), resolveTestManifest("nested", 10)},+ "sibling": {resolveTestData("sibling-data", 20)},+ "nested": {resolveTestData("nested-data", 10)},+ },+ map[string]time.Duration{"parent": 60 * time.Millisecond, "sibling": 5 * time.Millisecond},+ )++ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("parent", 0),+ resolveTestManifest("sibling", 20),+ }, 0, 100, nil)+ require.NoError(t, err)+ require.Equal(t, []string{"parent-data", "nested-data", "sibling-data"}, fileIDs(data))+ require.Equal(t, []string{"parent", "nested", "sibling"}, fileIDs(meta))+ require.Equal(t, []string{"sibling", "parent", "nested"}, fixture.completionOrder())+}++func TestResolveChunkManifestFiltersBeforeReadingAndRecursively(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "outside": {resolveTestData("never-read", 0)},+ "inside": {resolveTestData("before", 0), resolveTestData("selected", 100), resolveTestData("after", 200)},+ }, nil)++ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("outside", 0),+ resolveTestManifest("inside", 90),+ }, 100, 200, nil)+ require.NoError(t, err)+ require.Equal(t, []string{"selected"}, fileIDs(data))+ require.Equal(t, []string{"inside"}, fileIDs(meta))+ require.Equal(t, int32(1), fixture.loads.Load(), "the non-overlapping parent manifest must not be read")+}++func TestResolveChunkManifestPropagatesReadAndFormatErrorsInInputOrder(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "valid": {resolveTestData("valid-data", 0)},+ "broken": nil,+ }, map[string]time.Duration{"broken": 50 * time.Millisecond})+ fixture.manifests["broken"] = []byte("not a protobuf manifest")++ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestData("plain", 0),+ resolveTestManifest("valid", 10),+ resolveTestManifest("broken", 20),+ }, 0, 100, nil)+ require.Error(t, err)+ require.Nil(t, meta)+ require.Equal(t, []string{"plain", "valid-data"}, fileIDs(data))+ require.Contains(t, err.Error(), "fail to unmarshal manifest broken")++ lookupErr := errors.New("lookup failed")+ lookup := func(context.Context, string) ([]string, error) { return nil, lookupErr }+ _, _, err = ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{resolveTestManifest("lookup-error", 0)}, 0, 100, nil)+ require.ErrorIs(t, err, lookupErr)+}++func TestResolveChunkManifestCancellationStopsAllReads(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "one": {resolveTestData("one-data", 0)},+ "two": {resolveTestData("two-data", 10)},+ },+ map[string]time.Duration{"one": time.Second, "two": time.Second},+ )+ ctx, cancel := context.WithCancel(context.Background())+ result := make(chan error, 1)+ go func() {+ _, _, err := ResolveChunkManifest(ctx, fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("one", 0),+ resolveTestManifest("two", 10),+ }, 0, 100, nil)+ result <- err+ }()++ deadline := time.Now().Add(time.Second)+ for fixture.loads.Load() < 1 && time.Now().Before(deadline) {+ time.Sleep(time.Millisecond)+ }+ close(fixture.stopReads)+ cancel()+ err := <-result+ require.ErrorIs(t, err, context.Canceled)+ require.Eventually(t, func() bool {+ return fixture.active.Load() == 0+ }, time.Second, time.Millisecond, "all manifest readers must exit after ResolveChunkManifest returns")+}++type manifestFailureReadFixture struct {+ server *httptest.Server+ fastRelease chan struct{}+ slowStarted chan struct{}+ fastStarted chan struct{}+ slowCanceled chan struct{}+}++func newManifestFailureReadFixture(t testing.TB) *manifestFailureReadFixture {+ t.Helper()+ fixture := &manifestFailureReadFixture{+ fastRelease: make(chan struct{}),+ slowStarted: make(chan struct{}),+ fastStarted: make(chan struct{}),+ slowCanceled: make(chan struct{}),+ }+ fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch strings.TrimPrefix(r.URL.Path, "/") {+ case "fast":+ close(fixture.fastStarted)+ <-fixture.fastRelease+ w.Header().Set("Content-Length", "7")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("invalid"))+ case "slow":+ close(fixture.slowStarted)+ <-r.Context().Done()+ close(fixture.slowCanceled)+ }+ }))+ t.Cleanup(fixture.server.Close)+ return fixture+}++func (f *manifestFailureReadFixture) lookup(ctx context.Context, fileID string) ([]string, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ return []string{f.server.URL + "/" + fileID}, nil+}++func waitForManifestFailureSignal(t *testing.T, signal <-chan struct{}) {+ t.Helper()+ select {+ case <-signal:+ case <-time.After(time.Second):+ t.Fatal("timed out waiting for manifest request")+ }+}++func TestResolveChunkManifestFastFailureCancelsSlowSibling(t *testing.T) {+ testCases := []struct {+ name string+ inputIDs []string+ expectedID string+ }{+ {name: "fast then slow returns quickly", inputIDs: []string{"fast", "slow"}, expectedID: "fast"},+ {name: "slow then fast keeps real error", inputIDs: []string{"slow", "fast"}, expectedID: "fast"},+ }++ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ fixture := newManifestFailureReadFixture(t)+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ chunks := make([]*filer_pb.FileChunk, 0, len(testCase.inputIDs))+ for i, id := range testCase.inputIDs {+ chunks = append(chunks, resolveTestManifest(id, int64(i*100)))+ }+ result := make(chan error, 1)+ go func() {+ _, _, err := ResolveChunkManifest(ctx, fixture.lookup, chunks, 0, 200, nil)+ result <- err+ }()++ waitForManifestFailureSignal(t, fixture.fastStarted)+ waitForManifestFailureSignal(t, fixture.slowStarted)+ close(fixture.fastRelease)++ var err error+ select {+ case err = <-result:+ case <-time.After(time.Second):+ cancel()+ select {+ case <-result:+ case <-time.After(time.Second):+ t.Fatal("ResolveChunkManifest did not finish after caller cancellation")+ }+ t.Fatal("fast manifest failure waited for the slow sibling")+ }+ require.Error(t, err)+ require.Contains(t, err.Error(), "fail to unmarshal manifest "+testCase.expectedID)+ require.NotErrorIs(t, err, context.Canceled)+ waitForManifestFailureSignal(t, fixture.slowCanceled)+ })+ }+}++func TestResolveChunkManifestFastFailureCancelsEncryptedSibling(t *testing.T) {+ fixture := newManifestFailureReadFixture(t)+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ slowChunk := resolveTestManifest("slow", 0)+ slowChunk.CipherKey = []byte("0123456789abcdef")+ chunks := []*filer_pb.FileChunk{+ slowChunk,+ resolveTestManifest("fast", 100),+ }+ result := make(chan error, 1)+ go func() {+ _, _, err := ResolveChunkManifest(ctx, fixture.lookup, chunks, 0, 200, nil)+ result <- err+ }()++ waitForManifestFailureSignal(t, fixture.fastStarted)+ waitForManifestFailureSignal(t, fixture.slowStarted)+ close(fixture.fastRelease)++ var err error+ select {+ case err = <-result:+ case <-time.After(time.Second):+ cancel()+ select {+ case <-result:+ case <-time.After(time.Second):+ t.Fatal("ResolveChunkManifest did not finish after caller cancellation")+ }+ t.Fatal("fast manifest failure waited for the encrypted slow sibling")+ }+ require.Error(t, err)+ require.Contains(t, err.Error(), "fail to unmarshal manifest fast")+ require.NotErrorIs(t, err, context.Canceled)+ waitForManifestFailureSignal(t, fixture.slowCanceled)+}++func TestResolveChunkManifestKeepsRealErrorAfterInternalCancellation(t *testing.T) {+ earlyStarted := make(chan struct{})+ lateStarted := make(chan struct{})+ earlyErr := errors.New("early lookup failed")+ lateErr := errors.New("late lookup failed")+ lookup := func(ctx context.Context, fileID string) ([]string, error) {+ switch fileID {+ case "early":+ close(earlyStarted)+ <-ctx.Done()+ return nil, earlyErr+ case "late":+ close(lateStarted)+ return nil, lateErr+ default:+ return nil, fmt.Errorf("unexpected manifest %s", fileID)+ }+ }++ result := make(chan error, 1)+ go func() {+ _, _, err := ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{+ resolveTestManifest("early", 0),+ resolveTestManifest("late", 100),+ }, 0, 200, nil)+ result <- err+ }()+ waitForManifestFailureSignal(t, earlyStarted)+ waitForManifestFailureSignal(t, lateStarted)++ select {+ case err := <-result:+ require.ErrorIs(t, err, earlyErr)+ require.NotErrorIs(t, err, lateErr)+ case <-time.After(time.Second):+ t.Fatal("ResolveChunkManifest did not return the input-order error")+ }+}++func TestResolveChunkManifestReturnsLaterFailureWithoutRecursingEarlierChildren(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "a": {resolveTestManifest("a-child", 0)},+ "a-child": {resolveTestData("a-child-data", 0)},+ "b": nil,+ },+ map[string]time.Duration{"a": 5 * time.Millisecond, "b": 50 * time.Millisecond, "a-child": 2 * time.Second},+ )+ fixture.manifests["b"] = []byte("not a protobuf manifest")++ start := time.Now()+ data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("a", 0),+ resolveTestManifest("b", 100),+ }, 0, 200, nil)+ elapsed := time.Since(start)++ require.Error(t, err)+ require.Contains(t, err.Error(), "fail to unmarshal manifest b")+ require.Empty(t, data, "earlier manifest's children must not be recursed when a later manifest already failed")+ require.Nil(t, meta)+ require.Equal(t, int32(2), fixture.loads.Load(), "only the two top-level manifests must be read, not a-child")+ require.Less(t, elapsed, time.Second, "must return promptly without waiting for a-child's slow read")+}++func TestResolveChunkManifestExcludesOutOfRangeChildrenOnLaterFailure(t *testing.T) {+ fixture := newManifestReadFixture(t,+ map[string][]*filer_pb.FileChunk{+ "a": {+ resolveTestData("a-in-range", 0),+ resolveTestData("a-out-of-range", 500),+ },+ "b": nil,+ },+ map[string]time.Duration{"b": 50 * time.Millisecond},+ )+ fixture.manifests["b"] = []byte("not a protobuf manifest")++ data, _, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{+ resolveTestManifest("a", 0),+ resolveTestManifest("b", 100),+ }, 0, 200, nil)+ require.Error(t, err)+ require.Contains(t, err.Error(), "fail to unmarshal manifest b")+ require.Equal(t, []string{"a-in-range"}, fileIDs(data), "out-of-range child chunks must be excluded from partial results")+}++func TestResolveChunkManifestQueuesJobsBeyondWorkerCount(t *testing.T) {+ // More manifests than workers. The first four stall; the fifth fails+ // promptly. With a buffered job channel, the fifth job is queued without+ // blocking submission. When a stalled read is released, a worker picks up+ // the failing job, which cancels the batch and unblocks the rest.+ stallRelease := make(chan struct{})+ stallStarted := make(chan struct{}, 4)+ lookup := func(ctx context.Context, fileID string) ([]string, error) {+ switch fileID {+ case "stall-0", "stall-1", "stall-2", "stall-3":+ stallStarted <- struct{}{}+ select {+ case <-stallRelease:+ return nil, nil+ case <-ctx.Done():+ return nil, ctx.Err()+ }+ case "fail":+ return nil, errors.New("prompt lookup failure")+ default:+ return nil, fmt.Errorf("unexpected manifest %s", fileID)+ }+ }++ result := make(chan error, 1)+ go func() {+ _, _, err := ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{+ resolveTestManifest("stall-0", 0),+ resolveTestManifest("stall-1", 100),+ resolveTestManifest("stall-2", 200),+ resolveTestManifest("stall-3", 300),+ resolveTestManifest("fail", 400),+ }, 0, 500, nil)+ result <- err+ }()++ // Wait until all 4 workers are stalled.+ for i := 0; i < 4; i++ {+ waitForManifestFailureSignal(t, stallStarted)+ }++ // Release the stalled reads so workers can pick up the queued failing job.+ close(stallRelease)++ select {+ case err := <-result:+ require.Error(t, err)+ require.Contains(t, err.Error(), "fail to read manifest fail")+ case <-time.After(2 * time.Second):+ t.Fatal("failing job was not picked up after releasing stalled workers")+ }+}++func fileIDs(chunks []*filer_pb.FileChunk) []string {+ ids := make([]string, 0, len(chunks))+ for _, chunk := range chunks {+ ids = append(ids, chunk.GetFileIdString())+ }+ return ids+}++func resolveChunkManifestSerialForBenchmark(ctx context.Context, lookupFileIdFn func(context.Context, string) ([]string, error), chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, err error) {+ for _, chunk := range chunks {+ if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {+ continue+ }+ if !chunk.IsChunkManifest {+ dataChunks = append(dataChunks, chunk)+ continue+ }++ resolvedChunks, resolveErr := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk, nil)+ if resolveErr != nil {+ return dataChunks, nil, resolveErr+ }+ manifestChunks = append(manifestChunks, chunk)+ subDataChunks, subManifestChunks, subErr := resolveChunkManifestSerialForBenchmark(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset)+ if subErr != nil {+ return dataChunks, nil, subErr+ }+ dataChunks = append(dataChunks, subDataChunks...)+ manifestChunks = append(manifestChunks, subManifestChunks...)+ }+ return dataChunks, manifestChunks, nil+}++func BenchmarkResolveChunkManifestSerialVsParallel(b *testing.B) {+ manifests := make(map[string][]*filer_pb.FileChunk)+ inputs := make([]*filer_pb.FileChunk, 0, 8)+ delays := make(map[string]time.Duration)+ for i := 0; i < 8; i++ {+ id := fmt.Sprintf("m%d", i)+ manifests[id] = []*filer_pb.FileChunk{resolveTestData(fmt.Sprintf("d%d", i), int64(i*10))}+ inputs = append(inputs, resolveTestManifest(id, int64(i*10)))+ delays[id] = time.Millisecond+ }+ fixture := newManifestReadFixture(b, manifests, delays)+ b.ReportAllocs()++ b.Run("serial", func(b *testing.B) {+ for i := 0; i < b.N; i++ {+ _, _, err := resolveChunkManifestSerialForBenchmark(context.Background(), fixture.lookup, inputs, 0, 1000)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ b.Run("parallel", func(b *testing.B) {+ for i := 0; i < b.N; i++ {+ _, _, err := ResolveChunkManifest(context.Background(), fixture.lookup, inputs, 0, 1000, nil)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+}diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.goindex ab75c2ad94..fba9225634 100644--- a/weed/util/http/http_global_client_util.go+++ b/weed/util/http/http_global_client_util.go@@ -98,11 +98,11 @@ func Post(url string, values url.Values) ([]byte, error) {// github.com/seaweedfs/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go// may need increasing http.Client.Timeoutfunc Get(url string) ([]byte, bool, error) {- return GetAuthenticated(url, "")+ return GetAuthenticated(context.Background(), url, "")}-func GetAuthenticated(url, jwt string) ([]byte, bool, error) {- request, err := http.NewRequest(http.MethodGet, url, nil)+func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error) {+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)if err != nil {return nil, true, err}@@ -111,7 +111,9 @@ func GetAuthenticated(url, jwt string) ([]byte, bool, error) {response, err := GetGlobalHttpClient().Do(request)if err != nil {- recordUnreachable(request.URL.Host)+ if ctx.Err() == nil {+ recordUnreachable(request.URL.Host)+ }return nil, true, err}recordReachable(request.URL.Host)@@ -135,6 +137,9 @@ func GetAuthenticated(url, jwt string) ([]byte, bool, error) {return nil, retryable, fmt.Errorf("%s: %s", url, response.Status)}if err != nil {+ if ctxErr := ctx.Err(); ctxErr != nil {+ return nil, false, ctxErr+ }return nil, false, err}return b, false, nil@@ -454,7 +459,7 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte,}func readEncryptedUrl(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {- encryptedData, retryable, err := GetAuthenticated(fileUrl, jwt)+ encryptedData, retryable, err := GetAuthenticated(ctx, fileUrl, jwt)if err != nil {return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)}