Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

10.4 KB
Raw Blame
package s3api

import (
	"bytes"
	"context"
	"io"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// TestCreateDestinationChunkPreservesEncryption tests that createDestinationChunk preserves CipherKey and IsCompressed
func TestCreateDestinationChunkPreservesEncryption(t *testing.T) {
	s3a := &S3ApiServer{}

	testCases := []struct {
		name             string
		sourceChunk      *filer_pb.FileChunk
		expectedOffset   int64
		expectedSize     uint64
		shouldPreserveCK bool
		shouldPreserveIC bool
	}{
		{
			name: "Encrypted and compressed chunk",
			sourceChunk: &filer_pb.FileChunk{
				Offset:       0,
				Size:         1024,
				CipherKey:    []byte("test-cipher-key-1234567890123456"),
				IsCompressed: true,
				ETag:         "test-etag",
			},
			expectedOffset:   0,
			expectedSize:     1024,
			shouldPreserveCK: true,
			shouldPreserveIC: true,
		},
		{
			name: "Only encrypted chunk",
			sourceChunk: &filer_pb.FileChunk{
				Offset:       1024,
				Size:         2048,
				CipherKey:    []byte("test-cipher-key-1234567890123456"),
				IsCompressed: false,
				ETag:         "test-etag-2",
			},
			expectedOffset:   1024,
			expectedSize:     2048,
			shouldPreserveCK: true,
			shouldPreserveIC: false,
		},
		{
			name: "Only compressed chunk",
			sourceChunk: &filer_pb.FileChunk{
				Offset:       2048,
				Size:         512,
				CipherKey:    nil,
				IsCompressed: true,
				ETag:         "test-etag-3",
			},
			expectedOffset:   2048,
			expectedSize:     512,
			shouldPreserveCK: false,
			shouldPreserveIC: true,
		},
		{
			name: "Unencrypted and uncompressed chunk",
			sourceChunk: &filer_pb.FileChunk{
				Offset:       4096,
				Size:         1024,
				CipherKey:    nil,
				IsCompressed: false,
				ETag:         "test-etag-4",
			},
			expectedOffset:   4096,
			expectedSize:     1024,
			shouldPreserveCK: false,
			shouldPreserveIC: false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			dstChunk := s3a.createDestinationChunk(tc.sourceChunk, tc.expectedOffset, tc.expectedSize)

			// Verify offset and size
			if dstChunk.Offset != tc.expectedOffset {
				t.Errorf("Expected offset %d, got %d", tc.expectedOffset, dstChunk.Offset)
			}
			if dstChunk.Size != tc.expectedSize {
				t.Errorf("Expected size %d, got %d", tc.expectedSize, dstChunk.Size)
			}

			// Verify CipherKey preservation
			if tc.shouldPreserveCK {
				if !bytes.Equal(dstChunk.CipherKey, tc.sourceChunk.CipherKey) {
					t.Errorf("CipherKey not preserved: expected %v, got %v", tc.sourceChunk.CipherKey, dstChunk.CipherKey)
				}
			} else {
				if len(dstChunk.CipherKey) > 0 {
					t.Errorf("Expected no CipherKey, got %v", dstChunk.CipherKey)
				}
			}

			// Verify IsCompressed preservation
			if dstChunk.IsCompressed != tc.shouldPreserveIC {
				t.Errorf("IsCompressed not preserved: expected %v, got %v", tc.shouldPreserveIC, dstChunk.IsCompressed)
			}

			// Verify ETag preservation
			if dstChunk.ETag != tc.sourceChunk.ETag {
				t.Errorf("ETag not preserved: expected %s, got %s", tc.sourceChunk.ETag, dstChunk.ETag)
			}
		})
	}
}

// TestEncryptedVolumeCopyScenario documents the expected behavior for encrypted volumes (issue #7530)
func TestEncryptedVolumeCopyScenario(t *testing.T) {
	t.Run("Scenario: Copy file on encrypted volume with multiple chunks", func(t *testing.T) {
		// Scenario description for issue #7530:
		// 1. Volume is started with -filer.encryptVolumeData
		// 2. File is uploaded via S3 (automatically encrypted, multiple chunks)
		// 3. File is copied/renamed via S3 CopyObject
		// 4. Copied file should be readable
		//
		// The bug was that IsCompressed flag was not preserved during copy,
		// causing the upload logic to potentially double-compress the data,
		// making the copied file unreadable.

		sourceChunks := []*filer_pb.FileChunk{
			{
				FileId:       "1,abc123",
				Offset:       0,
				Size:         4194304,
				CipherKey:    util.GenCipherKey(), // Simulates encrypted volume
				IsCompressed: true,                // Simulates compression
				ETag:         "etag1",
			},
			{
				FileId:       "2,def456",
				Offset:       4194304,
				Size:         4194304,
				CipherKey:    util.GenCipherKey(),
				IsCompressed: true,
				ETag:         "etag2",
			},
		}

		s3a := &S3ApiServer{}

		// Verify that createDestinationChunk preserves all necessary metadata
		for i, srcChunk := range sourceChunks {
			dstChunk := s3a.createDestinationChunk(srcChunk, srcChunk.Offset, srcChunk.Size)

			// Critical checks for issue #7530
			if !dstChunk.IsCompressed {
				t.Errorf("Chunk %d: IsCompressed flag MUST be preserved to prevent double-compression", i)
			}
			if !bytes.Equal(dstChunk.CipherKey, srcChunk.CipherKey) {
				t.Errorf("Chunk %d: CipherKey MUST be preserved for encrypted volumes", i)
			}
			if dstChunk.Offset != srcChunk.Offset {
				t.Errorf("Chunk %d: Offset must be preserved", i)
			}
			if dstChunk.Size != srcChunk.Size {
				t.Errorf("Chunk %d: Size must be preserved", i)
			}
			if dstChunk.ETag != srcChunk.ETag {
				t.Errorf("Chunk %d: ETag must be preserved", i)
			}
		}

		t.Log("✓ All chunk metadata properly preserved for encrypted volume copy scenario")
	})
}

// A volume-encrypted source must take the re-encrypting UploadPartCopy path:
// the raw chunk copy slices ciphertext the destination's whole-chunk cipher key
// can no longer decrypt, and reports the part's ETag from chunks that carry
// none (issue #10968).
func TestSourceEntryIsEncryptedForVolumeCipher(t *testing.T) {
	testCases := []struct {
		name  string
		entry *filer_pb.Entry
		want  bool
	}{
		{
			name:  "nil entry",
			entry: nil,
		},
		{
			name: "plaintext chunks",
			entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
				{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
			}},
		},
		{
			name: "volume-encrypted chunk",
			entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
				{FileId: "1,abc123", Size: 1024, CipherKey: util.GenCipherKey()},
			}},
			want: true,
		},
		{
			name: "volume-encrypted second chunk",
			entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
				{FileId: "1,abc123", Size: 1024, ETag: "etag1"},
				{FileId: "2,def456", Offset: 1024, Size: 1024, CipherKey: util.GenCipherKey()},
			}},
			want: true,
		},
		{
			name: "SSE-S3 chunk",
			entry: &filer_pb.Entry{Chunks: []*filer_pb.FileChunk{
				{FileId: "1,abc123", Size: 1024, SseType: filer_pb.SSEType_SSE_S3},
			}},
			want: true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if got := sourceEntryIsEncrypted(tc.entry); got != tc.want {
				t.Errorf("sourceEntryIsEncrypted = %v, want %v", got, tc.want)
			}
		})
	}
}

// A ranged part copy asks the chunk stream for its slice; reading the whole
// object and discarding the prefix would make an N-part copy read the source
// N/2 times over.
func TestGetEncryptedStreamFromVolumesRangesInlineContent(t *testing.T) {
	s3a := &S3ApiServer{}
	entry := &filer_pb.Entry{Content: []byte("0123456789")}

	testCases := []struct {
		name   string
		offset int64
		size   int64
		want   string
	}{
		{name: "whole content", size: 10, want: "0123456789"},
		{name: "leading slice", size: 4, want: "0123"},
		{name: "middle slice", offset: 3, size: 4, want: "3456"},
		{name: "trailing slice", offset: 6, size: 4, want: "6789"},
		{name: "size past the end", offset: 8, size: 10, want: "89"},
		{name: "offset past the end", offset: 10, size: 4},
		{name: "empty range", size: 0},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			reader, err := s3a.getEncryptedStreamFromVolumes(context.Background(), entry, tc.offset, tc.size)
			if err != nil {
				t.Fatalf("getEncryptedStreamFromVolumes: %v", err)
			}
			defer reader.Close()
			got, err := io.ReadAll(reader)
			if err != nil {
				t.Fatalf("read: %v", err)
			}
			if string(got) != tc.want {
				t.Errorf("got %q, want %q", got, tc.want)
			}
		})
	}
}

func TestDetermineUnifiedCopyStrategySSES3(t *testing.T) {
	testCases := []struct {
		name     string
		state    *EncryptionState
		expected UnifiedCopyStrategy
	}{
		{
			name: "SSE-S3 to SSE-S3 direct copy",
			state: &EncryptionState{
				SrcSSES3: true,
				DstSSES3: true,
			},
			expected: CopyStrategyDirect,
		},
		{
			name: "SSE-S3 to plain decrypt copy",
			state: &EncryptionState{
				SrcSSES3: true,
				DstSSES3: false,
			},
			expected: CopyStrategyDecrypt,
		},
		{
			name: "Plain to SSE-S3 encrypt copy",
			state: &EncryptionState{
				SrcSSES3: false,
				DstSSES3: true,
			},
			expected: CopyStrategyEncrypt,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			strategy, err := DetermineUnifiedCopyStrategy(tc.state, nil, nil)
			if err != nil {
				t.Fatalf("DetermineUnifiedCopyStrategy failed: %v", err)
			}
			if strategy != tc.expected {
				t.Errorf("expected strategy %v, got %v", tc.expected, strategy)
			}
		})
	}
}

func TestDecryptChunkVolumeCipher(t *testing.T) {
	s3a := &S3ApiServer{}
	plainData := []byte("hello-seaweedfs-encrypted-volume-data-verification-test")

	t.Run("unencrypted chunk passes through", func(t *testing.T) {
		chunk := &filer_pb.FileChunk{
			CipherKey: nil,
		}
		got, err := s3a.decryptChunkVolumeCipher(plainData, chunk)
		if err != nil {
			t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
		}
		if !bytes.Equal(got, plainData) {
			t.Fatalf("expected %q, got %q", plainData, got)
		}
	})

	t.Run("volume cipher encrypted chunk decrypts correctly", func(t *testing.T) {
		cipherKey := util.GenCipherKey()
		encrypted, err := util.Encrypt(plainData, cipherKey)
		if err != nil {
			t.Fatalf("Encrypt failed: %v", err)
		}

		chunk := &filer_pb.FileChunk{
			CipherKey: cipherKey,
		}
		got, err := s3a.decryptChunkVolumeCipher(encrypted, chunk)
		if err != nil {
			t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
		}
		if !bytes.Equal(got, plainData) {
			t.Fatalf("expected %q, got %q", plainData, got)
		}
	})

	t.Run("volume cipher encrypted and compressed chunk decrypts correctly", func(t *testing.T) {
		cipherKey := util.GenCipherKey()
		compressed, err := util.GzipData(plainData)
		if err != nil {
			t.Fatalf("GzipData failed: %v", err)
		}
		encrypted, err := util.Encrypt(compressed, cipherKey)
		if err != nil {
			t.Fatalf("Encrypt failed: %v", err)
		}

		chunk := &filer_pb.FileChunk{
			CipherKey:    cipherKey,
			IsCompressed: true,
		}
		got, err := s3a.decryptChunkVolumeCipher(encrypted, chunk)
		if err != nil {
			t.Fatalf("decryptChunkVolumeCipher failed: %v", err)
		}
		if !bytes.Equal(got, plainData) {
			t.Fatalf("expected %q, got %q", plainData, got)
		}
	})
}