Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3api: copy the trailer checksum before reading the next trailer line (#11583)

augboot39bc9cd0efe1
diff --git a/weed/s3api/chunked_reader_trailer_split_test.go b/weed/s3api/chunked_reader_trailer_split_test.go
new file mode 100644
index 0000000000..6d457657ca
--- /dev/null
+++ b/weed/s3api/chunked_reader_trailer_split_test.go
@@ -0,0 +1,59 @@
+package s3api
+
+import (
+ "bufio"
+ "crypto/sha256"
+ "encoding/base64"
+ "hash/crc32"
+ "io"
+ "testing"
+)
+
+// segmentedReader returns one segment per Read call.
+type segmentedReader struct{ segments [][]byte }
+
+func (s *segmentedReader) Read(p []byte) (int, error) {
+ if len(s.segments) == 0 {
+ return 0, io.EOF
+ }
+ n := copy(p, s.segments[0])
+ s.segments[0] = s.segments[0][n:]
+ if len(s.segments[0]) == 0 {
+ s.segments = s.segments[1:]
+ }
+ return n, nil
+}
+
+func TestTrailerChecksumSurvivesSplitTrailerLines(t *testing.T) {
+ payload := []byte("hello, trailer\n")
+ crcWriter := crc32.NewIEEE()
+ crcWriter.Write(payload)
+ checksum := base64.StdEncoding.EncodeToString(crcWriter.Sum(nil))
+ sig := "0000000000000000000000000000000000000000000000000000000000000000"
+
+ segments := [][]byte{
+ []byte("f;chunk-signature=" + sig + "\r\n"),
+ append(payload, "\r\n"...),
+ []byte("0;chunk-signature=" + sig + "\r\n"),
+ []byte("x-amz-checksum-crc32:" + checksum),
+ []byte("\r\n"),
+ []byte("x-amz-trailer-signature:" + sig + "\r\n\r\n"),
+ }
+
+ cr := &s3ChunkedReader{
+ reader: bufio.NewReader(&segmentedReader{segments: segments}),
+ chunkSHA256Writer: sha256.New(),
+ checkSumAlgorithm: ChecksumAlgorithmCRC32.String(),
+ checkSumWriter: getCheckSumWriter(ChecksumAlgorithmCRC32),
+ state: readChunkHeader,
+ hasTrailer: true,
+ }
+
+ got, err := io.ReadAll(cr)
+ if err != nil {
+ t.Fatalf("read failed: %v", err)
+ }
+ if string(got) != string(payload) {
+ t.Fatalf("payload = %q, want %q", got, payload)
+ }
+}
diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go
index 31c7119210..362b77ba6d 100644
--- a/weed/s3api/chunked_reader_v4.go
+++ b/weed/s3api/chunked_reader_v4.go
@@ -537,7 +537,7 @@ func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte, error) {
glog.V(3).Infof("multiple checksum headers found in trailer, using last: %s", key)
}
checksumAlgorithm = alg
- checksum = value
+ checksum = bytes.Clone(value)
}
// Ignore other trailer headers like x-amz-trailer-signature
}