SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package sqlite_cbs
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// These tests exercise SeaweedFS the way SQLite Cloud Backed Storage (CBS)
// drives a cloud bucket. CBS needs put/fetch/delete/list primitives and —
// critically — conditional PUT semantics so two clients cannot clobber each
// other's manifest update: put-if-absent (If-None-Match: *) for new objects
// and compare-and-swap (If-Match: "<etag>") for manifest rewrites. A failed
// precondition must return 412 and must not commit, even when conditional
// writes to the same key race. See sqlite.org/cloudsqlite/doc/trunk/www/module.wiki.
func cbsTestContext() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 30*time.Second)
}
func cbsBucketName(prefix string) string {
return fmt.Sprintf("cbs-%s-%d", prefix, time.Now().UnixNano())
}
func createCBSBucket(t *testing.T, client *s3.Client, bucket string) {
t.Helper()
ctx, cancel := cbsTestContext()
defer cancel()
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
require.NoError(t, err)
}
func putCBSTestObject(t *testing.T, client *s3.Client, bucket, key, body string) *s3.PutObjectOutput {
t.Helper()
ctx, cancel := cbsTestContext()
defer cancel()
out, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader([]byte(body)),
})
require.NoError(t, err)
require.NotNil(t, out.ETag, "PutObject must return the ETag CBS feeds back into If-Match")
return out
}
func getCBSTestObject(t *testing.T, client *s3.Client, bucket, key string) (etag string, body []byte) {
t.Helper()
ctx, cancel := cbsTestContext()
defer cancel()
out, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
require.NoError(t, err)
defer out.Body.Close()
body, err = io.ReadAll(out.Body)
require.NoError(t, err)
require.NotNil(t, out.ETag, "GetObject must return the ETag CBS feeds back into If-Match")
return aws.ToString(out.ETag), body
}
func isPreconditionFailed(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "PreconditionFailed" {
return true
}
var respErr *awshttp.ResponseError
return errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusPreconditionFailed
}
func isNotModified(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotModified" {
return true
}
var respErr *awshttp.ResponseError
return errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotModified
}
func isNoSuchKey(err error) bool {
var noSuchKey *types.NoSuchKey
if errors.As(err, &noSuchKey) {
return true
}
var apiErr smithy.APIError
return errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchKey"
}
// TestCBSContainerAndObjectPrimitives covers the storage primitives a CBS
// cloud module needs: create/destroy a container (bucket), put, fetch, list,
// head and delete named objects.
func TestCBSContainerAndObjectPrimitives(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("primitives")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
_, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
require.NoError(t, err, "container should be reachable right after creation")
// xPut: store named block objects under a database prefix.
blocks := map[string]string{
"mydb/manifest": "gen=1 blocks=block-000,block-001",
"mydb/block-000": "page data for block 0",
"mydb/block-001": "page data for block 1",
}
for key, body := range blocks {
putCBSTestObject(t, client, bucket, key, body)
}
// xFetch: read objects back by name.
for key, want := range blocks {
_, got := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, want, string(got), "fetched body for %s", key)
}
// xList: enumerate the container's objects under the database prefix.
listOut, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String("mydb/"),
})
require.NoError(t, err)
gotKeys := make([]string, 0, len(listOut.Contents))
for _, obj := range listOut.Contents {
gotKeys = append(gotKeys, aws.ToString(obj.Key))
}
assert.ElementsMatch(t, []string{"mydb/manifest", "mydb/block-000", "mydb/block-001"}, gotKeys)
// A conditional fetch (xFetch carries an ETag) should answer 304 when the
// cached copy is current, so a CBS daemon revalidating a cached block does
// not re-download it.
manifestETag, _ := getCBSTestObject(t, client, bucket, "mydb/manifest")
getOut, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String("mydb/manifest"),
IfNoneMatch: aws.String(manifestETag),
})
if getOut != nil && getOut.Body != nil {
getOut.Body.Close()
}
require.Error(t, err, "If-None-Match with the current ETag must not return the object again")
assert.True(t, isNotModified(err), "expected 304 NotModified, got %v", err)
// xDelete: drop a named object.
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String("mydb/block-001"),
})
require.NoError(t, err)
_, err = client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String("mydb/block-001"),
})
require.Error(t, err, "deleted block should be gone")
// xDestroy: remove the remaining objects and the container itself.
for _, key := range []string{"mydb/manifest", "mydb/block-000"} {
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
require.NoError(t, err)
}
_, err = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
require.NoError(t, err)
}
// TestCBSPutIfAbsent covers the put-if-absent half of CBS's conditional-PUT
// requirement: If-None-Match: * must create the object when absent and fail
// with 412 without committing when the name is taken.
func TestCBSPutIfAbsent(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("if-absent")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
key := "mydb/block-007"
first, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader([]byte("first")),
})
require.NoError(t, err, "put-if-absent on a new block name must succeed")
require.NotNil(t, first.ETag)
second, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader([]byte("second")),
})
require.Error(t, err, "put-if-absent on a taken block name must fail")
assert.True(t, isPreconditionFailed(err), "expected 412 PreconditionFailed, got %v", err)
assert.Nil(t, second)
_, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, "first", string(body), "the rejected conditional PUT must not have committed")
}
// TestCBSCompareAndSwap covers the compare-and-swap half: If-Match carries the
// version identifier from a prior PUT/GET and the write only commits when the
// stored object is still exactly that version.
func TestCBSCompareAndSwap(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("cas")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
key := "mydb/manifest"
v1 := putCBSTestObject(t, client, bucket, key, "generation=1")
etag1 := aws.ToString(v1.ETag)
// If-Match with the version identifier just returned commits.
v2, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: aws.String(etag1),
Body: bytes.NewReader([]byte("generation=2")),
})
require.NoError(t, err)
etag2 := aws.ToString(v2.ETag)
require.NotEqual(t, etag1, etag2)
// A retry of the same conditional write — the identifier is now stale —
// must fail instead of clobbering generation 2.
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: aws.String(etag1),
Body: bytes.NewReader([]byte("generation=1-replayed")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err), "stale If-Match must yield 412, got %v", err)
// The current identifier still works.
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: aws.String(etag2),
Body: bytes.NewReader([]byte("generation=3")),
})
require.NoError(t, err)
_, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, "generation=3", string(body))
// If-Match on a name that does not exist can never commit — there is no
// version it could equal. SeaweedFS deliberately answers 404 NoSuchKey
// here rather than 412 (see validateConditionalHeaders); CBS only needs
// the write to fail, so accept either rejection.
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String("mydb/no-such-block"),
IfMatch: aws.String(etag2),
Body: bytes.NewReader([]byte("orphan")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err) || isNoSuchKey(err), "If-Match on an absent key must be rejected, got %v", err)
// If-Match: * is "any version exists": commits over an existing object,
// fails on an absent one (404 NoSuchKey here, same as above).
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: aws.String("*"),
Body: bytes.NewReader([]byte("generation=4")),
})
require.NoError(t, err)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String("mydb/another-missing-block"),
IfMatch: aws.String("*"),
Body: bytes.NewReader([]byte("orphan")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err) || isNoSuchKey(err), "If-Match:* on an absent key must be rejected, got %v", err)
// If-None-Match with a concrete identifier fails only when it equals the
// current version — the form a client uses for "put unless unchanged".
currentETag, _ := getCBSTestObject(t, client, bucket, key)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfNoneMatch: aws.String(currentETag),
Body: bytes.NewReader([]byte("unchanged rewrite")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err), "If-None-Match with the current ETag must yield 412, got %v", err)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfNoneMatch: aws.String(etag1),
Body: bytes.NewReader([]byte("generation=5")),
})
require.NoError(t, err, "If-None-Match with an outdated ETag must commit")
}
// TestCBSCompareAndSwapVersionedBucket runs the same CAS sequence on a
// versioned bucket. Conditional writes must validate against the latest
// version rather than a stale entry.
func TestCBSCompareAndSwapVersionedBucket(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("cas-versioned")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
_, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
key := "mydb/manifest"
v1 := putCBSTestObject(t, client, bucket, key, "generation=1")
v2 := putCBSTestObject(t, client, bucket, key, "generation=2")
// The latest version's identifier commits; v1's is stale and must fail.
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: v1.ETag,
Body: bytes.NewReader([]byte("stale rewrite")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err), "If-Match against a non-latest version must yield 412, got %v", err)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: v2.ETag,
Body: bytes.NewReader([]byte("generation=3")),
})
require.NoError(t, err)
// The CAS also works fed from the ETag a GET returned, matching how CBS
// re-reads the manifest before rewriting it.
etag3, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, "generation=3", string(body))
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: aws.String(etag3),
Body: bytes.NewReader([]byte("generation=4")),
})
require.NoError(t, err)
}
// TestCBSConcurrentPutIfAbsent races several CBS writers creating the same new
// block name: exactly one put-if-absent may commit.
func TestCBSConcurrentPutIfAbsent(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
bucket := cbsBucketName("race-absent")
adminClient := cluster.newS3Client(t)
createCBSBucket(t, adminClient, bucket)
const writers = 8
key := "mydb/block-contended"
start := make(chan struct{})
type putResult struct {
writer int
err error
}
results := make(chan putResult, writers)
var wg sync.WaitGroup
for i := 0; i < writers; i++ {
wg.Add(1)
// Separate clients stand in for separate CBS client processes.
writerClient := cluster.newS3Client(t)
go func(i int) {
defer wg.Done()
<-start
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := writerClient.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader([]byte(fmt.Sprintf("writer-%d", i))),
})
results <- putResult{i, err}
}(i)
}
close(start)
wg.Wait()
close(results)
successes, preconditionFailures, winner := 0, 0, -1
for res := range results {
switch {
case res.err == nil:
successes++
winner = res.writer
case isPreconditionFailed(res.err):
preconditionFailures++
default:
t.Fatalf("unexpected race error: %v", res.err)
}
}
require.Equal(t, 1, successes, "exactly one put-if-absent may commit")
require.Equal(t, writers-1, preconditionFailures, "every losing write must see 412")
// A losing write must not have committed over the winner's data.
_, body := getCBSTestObject(t, adminClient, bucket, key)
assert.Equal(t, fmt.Sprintf("writer-%d", winner), string(body))
}
// cbsManifest mirrors the state a CBS daemon keeps in cloud storage: a single
// object naming the current generation of database blocks, rewritten
// transactionally at each checkpoint.
type cbsManifest struct {
Updates []string `json:"updates"`
}
// TestCBSConcurrentManifestCAS is the core CBS correctness scenario: many
// clients read-modify-write the same manifest object. Every writer retries its
// read + If-Match PUT on 412, so all commits land and no update is silently
// lost — which is exactly what keeps the database intact when two cloud
// clients checkpoint at the same time.
func TestCBSConcurrentManifestCAS(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
bucket := cbsBucketName("manifest-cas")
adminClient := cluster.newS3Client(t)
createCBSBucket(t, adminClient, bucket)
const writers = 8
key := "mydb/manifest"
// Seed the manifest so every writer takes the read-then-CAS path.
putCBSTestObject(t, adminClient, bucket, key, `{"updates":[]}`)
start := make(chan struct{})
errs := make(chan error, writers)
var wg sync.WaitGroup
for i := 0; i < writers; i++ {
wg.Add(1)
writerTag := fmt.Sprintf("writer-%d", i)
writerClient := cluster.newS3Client(t)
go func() {
defer wg.Done()
<-start
errs <- casManifestUpdate(writerClient, bucket, key, writerTag)
}()
}
close(start)
wg.Wait()
close(errs)
for err := range errs {
require.NoError(t, err)
}
// Every committed update must be present exactly once: a lost CAS would
// drop a checkpoint's block list and corrupt the database.
finalETag, body := getCBSTestObject(t, adminClient, bucket, key)
require.NotEmpty(t, finalETag)
var manifest cbsManifest
require.NoError(t, json.Unmarshal(body, &manifest))
assert.Len(t, manifest.Updates, writers)
for i := 0; i < writers; i++ {
assert.Contains(t, manifest.Updates, fmt.Sprintf("writer-%d", i))
}
}
// casManifestUpdate performs one CBS-style manifest rewrite: fetch the current
// manifest and its version identifier, append the writer's tag, and commit
// with If-Match. A 412 means another client committed first — re-read and
// retry, which is what a CBS client must do on a failed conditional PUT.
func casManifestUpdate(client *s3.Client, bucket, key, tag string) error {
for attempt := 0; attempt < 64; attempt++ {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
getOut, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
cancel()
return fmt.Errorf("fetch manifest: %w", err)
}
raw, err := io.ReadAll(getOut.Body)
getOut.Body.Close()
if err != nil {
cancel()
return fmt.Errorf("read manifest: %w", err)
}
var manifest cbsManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
cancel()
return fmt.Errorf("parse manifest: %w", err)
}
manifest.Updates = append(manifest.Updates, tag)
updated, err := json.Marshal(manifest)
if err != nil {
cancel()
return err
}
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
IfMatch: getOut.ETag,
Body: bytes.NewReader(updated),
})
cancel()
switch {
case err == nil:
return nil
case isPreconditionFailed(err):
continue // another client committed first; re-read and retry
default:
return fmt.Errorf("commit manifest: %w", err)
}
}
return fmt.Errorf("writer %s never won the manifest CAS", tag)
}
// TestCBSConditionalMultipartCommit covers conditional headers on
// CompleteMultipartUpload. CBS daemons stage large uploads; the completion
// must apply the same conditions atomically with the publish, or two racing
// completions could both report success.
func TestCBSConditionalMultipartCommit(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("mp-cond")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
key := "mydb/big-block"
// If-None-Match: * completes when the object is absent.
uploadID, partETag := startMultipartUpload(t, client, bucket, key, "multipart-generation-1")
completeOut, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID),
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
{ETag: aws.String(partETag), PartNumber: aws.Int32(1)},
}},
IfNoneMatch: aws.String("*"),
})
require.NoError(t, err)
require.NotNil(t, completeOut.ETag, "CompleteMultipartUpload must return the ETag the If-Match checks feed back")
// A second conditional completion on the now-existing key must fail
// instead of silently replacing the object.
uploadID2, partETag2 := startMultipartUpload(t, client, bucket, key, "multipart-generation-2")
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID2),
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
{ETag: aws.String(partETag2), PartNumber: aws.Int32(1)},
}},
IfNoneMatch: aws.String("*"),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err), "expected 412 on conditional multipart completion, got %v", err)
abortMultipartUpload(client, bucket, key, uploadID2)
// If-Match commits when fed the stored version identifier, and the stale
// identifier from before must then fail.
uploadID3, partETag3 := startMultipartUpload(t, client, bucket, key, "multipart-generation-3")
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID3),
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
{ETag: aws.String(partETag3), PartNumber: aws.Int32(1)},
}},
IfMatch: completeOut.ETag,
})
require.NoError(t, err)
uploadID4, partETag4 := startMultipartUpload(t, client, bucket, key, "multipart-generation-4")
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID4),
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
{ETag: aws.String(partETag4), PartNumber: aws.Int32(1)},
}},
IfMatch: completeOut.ETag,
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err), "stale If-Match on multipart completion must yield 412, got %v", err)
abortMultipartUpload(client, bucket, key, uploadID4)
_, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, "multipart-generation-3", string(body))
}
// TestCBSConcurrentConditionalMultipartCommit races two completions of the
// same key, each guarded by put-if-absent. The race the post-4.51 work fixed:
// previously both completions could report success.
func TestCBSConcurrentConditionalMultipartCommit(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
bucket := cbsBucketName("mp-race")
adminClient := cluster.newS3Client(t)
createCBSBucket(t, adminClient, bucket)
const racers = 4
key := "mydb/raced-block"
uploadIDs := make([]string, racers)
partETags := make([]string, racers)
for i := 0; i < racers; i++ {
uploadIDs[i], partETags[i] = startMultipartUpload(t, adminClient, bucket, key, fmt.Sprintf("racer-%d", i))
}
start := make(chan struct{})
type raceResult struct {
racer int
err error
}
results := make(chan raceResult, racers)
var wg sync.WaitGroup
for i := 0; i < racers; i++ {
wg.Add(1)
racerClient := cluster.newS3Client(t)
uploadID, partETag := uploadIDs[i], partETags[i]
go func(i int) {
defer wg.Done()
<-start
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := racerClient.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID),
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
{ETag: aws.String(partETag), PartNumber: aws.Int32(1)},
}},
IfNoneMatch: aws.String("*"),
})
results <- raceResult{i, err}
}(i)
}
close(start)
wg.Wait()
close(results)
successes, preconditionFailures, winner := 0, 0, -1
for res := range results {
switch {
case res.err == nil:
successes++
winner = res.racer
case isPreconditionFailed(res.err):
preconditionFailures++
default:
t.Fatalf("unexpected race error: %v", res.err)
}
}
require.Equal(t, 1, successes, "exactly one conditional multipart completion may commit")
require.Equal(t, racers-1, preconditionFailures, "every losing completion must see 412")
// A losing completion must not have published over the winner's data.
_, body := getCBSTestObject(t, adminClient, bucket, key)
assert.Equal(t, fmt.Sprintf("racer-%d", winner), string(body))
for _, uploadID := range uploadIDs {
abortMultipartUpload(adminClient, bucket, key, uploadID)
}
}
// TestCBSDatabaseUploadDownloadRoundtrip replays a bcv-style session end to
// end: a client uploads immutable database blocks plus a manifest guarded by
// put-if-absent, a second client lists and fetches the database back, then a
// checkpoint writes new blocks and rewrites the manifest via CAS while a stale
// writer's CAS is rejected.
func TestCBSDatabaseUploadDownloadRoundtrip(t *testing.T) {
if testing.Short() {
t.Skip("skipping sqlite cbs integration test in short mode")
}
cluster := startCBSMiniCluster(t)
client := cluster.newS3Client(t)
bucket := cbsBucketName("roundtrip")
createCBSBucket(t, client, bucket)
ctx, cancel := cbsTestContext()
defer cancel()
prefix := "shop.db/"
manifestKey := prefix + "manifest"
const blockCount = 6
blockBody := func(i int) []byte {
return bytes.Repeat([]byte{byte('a' + i)}, 1024) // deterministic content per block
}
blockKey := func(i int) string {
return fmt.Sprintf("%sblock-%06d", prefix, i)
}
// Upload generation 1: immutable blocks with put-if-absent, then the
// manifest guarded the same way so no existing database is clobbered.
manifest := cbsManifest{Updates: make([]string, 0, blockCount)}
for i := 0; i < blockCount; i++ {
_, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(blockKey(i)),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader(blockBody(i)),
})
require.NoError(t, err, "upload block %d", i)
manifest.Updates = append(manifest.Updates, blockKey(i))
}
manifestBody, err := json.Marshal(manifest)
require.NoError(t, err)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(manifestKey),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader(manifestBody),
})
require.NoError(t, err)
// Re-uploading the same database under the same manifest name must not
// commit over generation 1.
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(manifestKey),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader([]byte("different manifest")),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err))
// Download: list the container, fetch the manifest, fetch and verify every
// block it names.
listOut, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
})
require.NoError(t, err)
assert.Len(t, listOut.Contents, blockCount+1, "manifest plus %d blocks", blockCount)
manifestETag, rawManifest := getCBSTestObject(t, client, bucket, manifestKey)
var fetched cbsManifest
require.NoError(t, json.Unmarshal(rawManifest, &fetched))
require.Len(t, fetched.Updates, blockCount)
for i, key := range fetched.Updates {
_, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, sha256.Sum256(blockBody(i)), sha256.Sum256(body), "block %s checksum", key)
}
// Checkpoint: append a new block, rewrite the manifest by CAS on the
// fetched ETag. A reader racing us with the old identifier loses.
newBlock := fmt.Sprintf("%sblock-%06d", prefix, blockCount)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(newBlock),
IfNoneMatch: aws.String("*"),
Body: bytes.NewReader(blockBody(blockCount)),
})
require.NoError(t, err)
updated := fetched
updated.Updates = append(updated.Updates, newBlock)
updatedBody, err := json.Marshal(updated)
require.NoError(t, err)
// A writer holding the generation-1 identifier loses the race...
stale, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(manifestKey),
IfMatch: aws.String("\"stale-generation-etag\""),
Body: bytes.NewReader(updatedBody),
})
require.Error(t, err)
assert.True(t, isPreconditionFailed(err))
assert.Nil(t, stale)
// ...while the writer that re-read the manifest commits.
gen2, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(manifestKey),
IfMatch: aws.String(manifestETag),
Body: bytes.NewReader(updatedBody),
})
require.NoError(t, err)
require.NotNil(t, gen2.ETag)
// The committed database is consistent: manifest lists every block and
// every block fetches intact.
_, rawManifest = getCBSTestObject(t, client, bucket, manifestKey)
require.NoError(t, json.Unmarshal(rawManifest, &fetched))
require.Len(t, fetched.Updates, blockCount+1)
for i, key := range fetched.Updates[:blockCount] {
_, body := getCBSTestObject(t, client, bucket, key)
assert.Equal(t, sha256.Sum256(blockBody(i)), sha256.Sum256(body))
}
_, body := getCBSTestObject(t, client, bucket, newBlock)
assert.Equal(t, sha256.Sum256(blockBody(blockCount)), sha256.Sum256(body))
// Tear the database down the way bcv destroy does.
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(manifestKey)})
require.NoError(t, err)
deleteOut, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{Objects: []types.ObjectIdentifier{
{Key: aws.String(blockKey(0))},
{Key: aws.String(newBlock)},
}},
})
require.NoError(t, err)
assert.Len(t, deleteOut.Deleted, 2)
}
func startMultipartUpload(t *testing.T, client *s3.Client, bucket, key, body string) (uploadID string, partETag string) {
t.Helper()
ctx, cancel := cbsTestContext()
defer cancel()
createOut, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
require.NoError(t, err)
require.NotNil(t, createOut.UploadId)
partOut, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: createOut.UploadId,
PartNumber: aws.Int32(1),
Body: bytes.NewReader([]byte(body)),
})
require.NoError(t, err)
require.NotNil(t, partOut.ETag)
return aws.ToString(createOut.UploadId), aws.ToString(partOut.ETag)
}
func abortMultipartUpload(client *s3.Client, bucket, key, uploadID string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, _ = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
UploadId: aws.String(uploadID),
})
}