SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package s3api
import (
"bytes"
"context"
"io"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// TestCreateDestinationChunkPreservesEncryption tests that createDestinationChunk preserves CipherKey and IsCompressed
func TestCreateDestinationChunkPreservesEncryption(t *testing.T) {
s3a := &S3ApiServer{}
testCases := []struct {
name string
sourceChunk *filer_pb.FileChunk
expectedOffset int64
expectedSize uint64
shouldPreserveCK bool
shouldPreserveIC bool
}{
{
name: "Encrypted and compressed chunk",
sourceChunk: &filer_pb.FileChunk{
Offset: 0,
Size: 1024,
CipherKey: []byte("test-cipher-key-1234567890123456"),
IsCompressed: true,
ETag: "test-etag",
},
expectedOffset: 0,
expectedSize: 1024,
shouldPreserveCK: true,
shouldPreserveIC: true,
},
{
name: "Only encrypted chunk",
sourceChunk: &filer_pb.FileChunk{
Offset: 1024,
Size: 2048,
CipherKey: []byte("test-cipher-key-1234567890123456"),
IsCompressed: false,
ETag: "test-etag-2",
},
expectedOffset: 1024,
expectedSize: 2048,
shouldPreserveCK: true,
shouldPreserveIC: false,
},
{
name: "Only compressed chunk",
sourceChunk: &filer_pb.FileChunk{
Offset: 2048,
Size: 512,
CipherKey: nil,
IsCompressed: true,
ETag: "test-etag-3",
},
expectedOffset: 2048,
expectedSize: 512,
shouldPreserveCK: false,
shouldPreserveIC: true,
},
{
name: "Unencrypted and uncompressed chunk",
sourceChunk: &filer_pb.FileChunk{
Offset: 4096,
Size: 1024,
CipherKey: nil,
IsCompressed: false,
ETag: "test-etag-4",
},
expectedOffset: 4096,
expectedSize: 1024,
shouldPreserveCK: false,
shouldPreserveIC: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
dstChunk := s3a.createDestinationChunk(tc.sourceChunk, tc.expectedOffset, tc.expectedSize)
// Verify offset and size
if dstChunk.Offset != tc.expectedOffset {
t.Errorf("Expected offset %d, got %d", tc.expectedOffset, dstChunk.Offset)
}
if dstChunk.Size != tc.expectedSize {
t.Errorf("Expected size %d, got %d", tc.expectedSize, dstChunk.Size)
}
// Verify CipherKey preservation
if tc.shouldPreserveCK {
if !bytes.Equal(dstChunk.CipherKey, tc.sourceChunk.CipherKey) {
t.Errorf("CipherKey not preserved: expected %v, got %v", tc.sourceChunk.CipherKey, dstChunk.CipherKey)
}
} else {
if len(dstChunk.CipherKey) > 0 {
t.Errorf("Expected no CipherKey, got %v", dstChunk.CipherKey)
}
}
// Verify IsCompressed preservation
if dstChunk.IsCompressed != tc.shouldPreserveIC {
t.Errorf("IsCompressed not preserved: expected %v, got %v", tc.shouldPreserveIC, dstChunk.IsCompressed)
}
// Verify ETag preservation
if dstChunk.ETag != tc.sourceChunk.ETag {
t.Errorf("ETag not preserved: expected %s, got %s", tc.sourceChunk.ETag, dstChunk.ETag)
}
})
}
}
// TestEncryptedVolumeCopyScenario documents the expected behavior for encrypted volumes (issue #7530)
func TestEncryptedVolumeCopyScenario(t *testing.T) {
t.Run("Scenario: Copy file on encrypted volume with multiple chunks", func(t *testing.T) {
// Scenario description for issue #7530:
// 1. Volume is started with -filer.encryptVolumeData
// 2. File is uploaded via S3 (automatically encrypted, multiple chunks)
// 3. File is copied/renamed via S3 CopyObject
// 4. Copied file should be readable
//
// The bug was that IsCompressed flag was not preserved during copy,
// causing the upload logic to potentially double-compress the data,
// making the copied file unreadable.
sourceChunks := []*filer_pb.FileChunk{
{
FileId: "1,abc123",
Offset: 0,
Size: 4194304,
CipherKey: util.GenCipherKey(), // Simulates encrypted volume
IsCompressed: true, // Simulates compression
ETag: "etag1",
},
{
FileId: "2,def456",
Offset: 4194304,
Size: 4194304,
CipherKey: util.GenCipherKey(),
IsCompressed: true,
ETag: "etag2",
},
}
s3a := &S3ApiServer{}
// Verify that createDestinationChunk preserves all necessary metadata
for i, srcChunk := range sourceChunks {
dstChunk := s3a.createDestinationChunk(srcChunk, srcChunk.Offset, srcChunk.Size)
// Critical checks for issue #7530
if !dstChunk.IsCompressed {
t.Errorf("Chunk %d: IsCompressed flag MUST be preserved to prevent double-compression", i)
}
if !bytes.Equal(dstChunk.CipherKey, srcChunk.CipherKey) {
t.Errorf("Chunk %d: CipherKey MUST be preserved for encrypted volumes", i)
}
if dstChunk.Offset != srcChunk.Offset {
t.Errorf("Chunk %d: Offset must be preserved", i)
}
if dstChunk.Size != srcChunk.Size {
t.Errorf("Chunk %d: Size must be preserved", i)
}
if dstChunk.ETag != srcChunk.ETag {
t.Errorf("Chunk %d: ETag must be preserved", i)
}
}
t.Log("✓ All chunk metadata properly preserved for encrypted volume copy scenario")
})
}
// A volume-encrypted source must take the re-encrypting UploadPartCopy path:
// the raw chunk copy slices ciphertext the destination's whole-chunk cipher key
// can no longer decrypt, and reports the part's ETag from chunks that carry
// none (issue #10968).
func TestSourceEntryIsEncryptedForVolumeCipher(t *testing.T) {
testCases := []struct {
name string
entry *filer_pb.Entry
want bool
}{
{
name: "nil entry",
entry: nil,
},
{
name: "plaintext chunks",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
}},
},
{
name: "volume-encrypted chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, CipherKey: util.GenCipherKey()},
}},
want: true,
},
{
name: "volume-encrypted second chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
{FileId: "2,def456", Offset: 1024, Size: 1024, CipherKey: util.GenCipherKey()},
}},
want: true,
},
{
name: "SSE-S3 chunk",
entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
{FileId: "1,abc123", Size: 1024, SseType: filer_pb.SSEType_SSE_S3},
}},
want: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if got := sourceEntryIsEncrypted(tc.entry); got != tc.want {
t.Errorf("sourceEntryIsEncrypted = %v, want %v", got, tc.want)
}
})
}
}
// A ranged part copy asks the chunk stream for its slice; reading the whole
// object and discarding the prefix would make an N-part copy read the source
// N/2 times over.
func TestGetEncryptedStreamFromVolumesRangesInlineContent(t *testing.T) {
s3a := &S3ApiServer{}
entry := &filer_pb.Entry{Content: []byte("0123456789")}
testCases := []struct {
name string
offset int64
size int64
want string
}{
{name: "whole content", size: 10, want: "0123456789"},
{name: "leading slice", size: 4, want: "0123"},
{name: "middle slice", offset: 3, size: 4, want: "3456"},
{name: "trailing slice", offset: 6, size: 4, want: "6789"},
{name: "size past the end", offset: 8, size: 10, want: "89"},
{name: "offset past the end", offset: 10, size: 4},
{name: "empty range", size: 0},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
reader, err := s3a.getEncryptedStreamFromVolumes(context.Background(), entry, tc.offset, tc.size)
if err != nil {
t.Fatalf("getEncryptedStreamFromVolumes: %v", err)
}
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("read: %v", err)
}
if string(got) != tc.want {
t.Errorf("got %q, want %q", got, tc.want)
}
})
}
}
func TestDetermineUnifiedCopyStrategySSES3(t *testing.T) {
testCases := []struct {
name string
state *EncryptionState
expected UnifiedCopyStrategy
}{
{
name: "SSE-S3 to SSE-S3 direct copy",
state: &EncryptionState{
SrcSSES3: true,
DstSSES3: true,
},
expected: CopyStrategyDirect,
},
{
name: "SSE-S3 to plain decrypt copy",
state: &EncryptionState{
SrcSSES3: true,
DstSSES3: false,
},
expected: CopyStrategyDecrypt,
},
{
name: "Plain to SSE-S3 encrypt copy",
state: &EncryptionState{
SrcSSES3: false,
DstSSES3: true,
},
expected: CopyStrategyEncrypt,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
strategy, err := DetermineUnifiedCopyStrategy(tc.state, nil, nil)
if err != nil {
t.Fatalf("DetermineUnifiedCopyStrategy failed: %v", err)
}
if strategy != tc.expected {
t.Errorf("expected strategy %v, got %v", tc.expected, strategy)
}
})
}
}
func TestDecryptChunkVolumeCipher(t *testing.T) {
s3a := &S3ApiServer{}
plainData := []byte("hello-seaweedfs-encrypted-volume-data-verification-test")
t.Run("unencrypted chunk passes through", func(t *testing.T) {
chunk := &filer_pb.FileChunk{
CipherKey: nil,
}
got, err := s3a.decryptChunkVolumeCipher(plainData, chunk)
if err != nil {
t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
}
if !bytes.Equal(got, plainData) {
t.Fatalf("expected %q, got %q", plainData, got)
}
})
t.Run("volume cipher encrypted chunk decrypts correctly", func(t *testing.T) {
cipherKey := util.GenCipherKey()
encrypted, err := util.Encrypt(plainData, cipherKey)
if err != nil {
t.Fatalf("Encrypt failed: %v", err)
}
chunk := &filer_pb.FileChunk{
CipherKey: cipherKey,
}
got, err := s3a.decryptChunkVolumeCipher(encrypted, chunk)
if err != nil {
t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
}
if !bytes.Equal(got, plainData) {
t.Fatalf("expected %q, got %q", plainData, got)
}
})
t.Run("volume cipher encrypted and compressed chunk decrypts correctly", func(t *testing.T) {
cipherKey := util.GenCipherKey()
compressed, err := util.GzipData(plainData)
if err != nil {
t.Fatalf("GzipData failed: %v", err)
}
encrypted, err := util.Encrypt(compressed, cipherKey)
if err != nil {
t.Fatalf("Encrypt failed: %v", err)
}
chunk := &filer_pb.FileChunk{
CipherKey: cipherKey,
IsCompressed: true,
}
got, err := s3a.decryptChunkVolumeCipher(encrypted, chunk)
if err != nil {
t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
}
if !bytes.Equal(got, plainData) {
t.Fatalf("expected %q, got %q", plainData, got)
}
})
}