Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3 sink: report the source read error the SDK hides (#11277)

Konstantin Adzer42b0ca785061
diff --git a/weed/filer/stream.go b/weed/filer/stream.go
index a910061f64..8642339c4b 100644
--- a/weed/filer/stream.go
+++ b/weed/filer/stream.go
@@ -369,6 +369,7 @@ type ChunkStreamReader struct {
bufferLock sync.Mutex
chunk string
lookupFileId wdclient.LookupFileIdFunctionType
+ sourceErr error
}
var _ = io.ReadSeeker(&ChunkStreamReader{})
@@ -507,7 +508,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
urlStrings, err := c.lookupFileId(context.Background(), chunkView.FileId)
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
- return err
+ return c.rememberSourceError(err)
}
var buffer bytes.Buffer
// pre-size to the known chunk size; avoids bytes.Buffer's doubling regrowth
@@ -529,7 +530,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
}
}
if err != nil {
- return err
+ return c.rememberSourceError(err)
}
c.buffer = buffer.Bytes()
c.bufferOffset = chunkView.ViewOffset
@@ -540,6 +541,28 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
return nil
}
+func (c *ChunkStreamReader) rememberSourceError(err error) error {
+ if c.sourceErr == nil {
+ c.sourceErr = err
+ }
+ return err
+}
+
+// SourceError returns the first lookup or chunk read failure, or nil.
+func (c *ChunkStreamReader) SourceError() error {
+ c.bufferLock.Lock()
+ defer c.bufferLock.Unlock()
+ return c.sourceErr
+}
+
+// ReaderSourceError returns the SourceError of a reader from NewFileReader, or nil.
+func ReaderSourceError(r io.Reader) error {
+ if csr, ok := r.(*ChunkStreamReader); ok {
+ return csr.SourceError()
+ }
+ return nil
+}
+
func (c *ChunkStreamReader) Close() error {
c.bufferLock.Lock()
defer c.bufferLock.Unlock()
diff --git a/weed/filer/stream_source_error_test.go b/weed/filer/stream_source_error_test.go
new file mode 100644
index 0000000000..cf6cb98be9
--- /dev/null
+++ b/weed/filer/stream_source_error_test.go
@@ -0,0 +1,55 @@
+package filer
+
+import (
+ "context"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestChunkStreamReaderSourceError(t *testing.T) {
+ server := createTestServer(map[string][]byte{})
+ defer server.Close()
+
+ lookup := func(ctx context.Context, fileId string) ([]string, error) {
+ return []string{server.URL + "/" + fileId}, nil
+ }
+ reader := NewChunkStreamReaderFromLookup(context.Background(), lookup,
+ []*filer_pb.FileChunk{{FileId: "7,01637037d6", Size: 8}})
+
+ if _, err := io.ReadAll(reader); err == nil {
+ t.Fatal("reading a gone chunk succeeded")
+ }
+ sourceErr := reader.SourceError()
+ if sourceErr == nil {
+ t.Fatal("reader kept no source error")
+ }
+ if !strings.Contains(sourceErr.Error(), "404") {
+ t.Fatalf("source error does not name the volume answer: %v", sourceErr)
+ }
+ if ReaderSourceError(reader) != sourceErr {
+ t.Fatal("ReaderSourceError does not return the kept error")
+ }
+}
+
+func TestChunkStreamReaderSourceErrorOnLookup(t *testing.T) {
+ master := &testMasterClient{urls: map[string][]string{}}
+ reader := NewChunkStreamReaderFromLookup(context.Background(), master.GetLookupFileIdFunction(),
+ []*filer_pb.FileChunk{{FileId: "7,01637037d6", Size: 8}})
+
+ if _, err := io.ReadAll(reader); err == nil {
+ t.Fatal("reading a chunk with no location succeeded")
+ }
+ if reader.SourceError() == nil {
+ t.Fatal("reader kept no source error")
+ }
+}
+
+func TestReaderSourceErrorInlineEntry(t *testing.T) {
+ reader := NewFileReader(nil, &filer_pb.Entry{Content: []byte("inline")})
+ if err := ReaderSourceError(reader); err != nil {
+ t.Fatalf("inline content reported a source error: %v", err)
+ }
+}
diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go
index 380a69d563..8f365a6e5a 100644
--- a/weed/replication/sink/s3sink/s3_sink.go
+++ b/weed/replication/sink/s3sink/s3_sink.go
@@ -220,6 +220,11 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
}
_, err = uploader.Upload(&uploadInput)
+ if err != nil {
+ if sourceErr := filer.ReaderSourceError(reader); sourceErr != nil {
+ return fmt.Errorf("read source %s: %w", key, sourceErr)
+ }
+ }
return err