Git designed for AI agents.
package offloading
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/feder-cr/git-ai/internal/backoff"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/stretchr/testify/require"
"gocloud.dev/blob"
"gocloud.dev/gcerrors"
)
var backoffStrategyInTest = constantBackoff{}
func TestNewSink(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
bucket := testhelper.TempDir(t)
localBucketURI := fmt.Sprintf("file://%s", bucket)
localBucket, err := blob.OpenBucket(ctx, localBucketURI)
require.NoError(t, err)
for _, tc := range []struct {
desc string
bucket Bucket
// allDefault, if true, all options are the default values.
allDefault bool
overallTimeout time.Duration
maxRetry uint
retryTimeout time.Duration
backoffStrategy backoff.Strategy
expectedError error
}{
{
desc: "create a new sink with all defaults",
bucket: localBucket,
allDefault: true,
expectedError: nil,
},
{
desc: "create a new sink with options",
bucket: localBucket,
overallTimeout: 15 * time.Second,
maxRetry: 0,
retryTimeout: 25 * time.Second,
backoffStrategy: defaultBackoffStrategy,
expectedError: nil,
},
{
// This is not an error case, no max retry means maxRetry == 0 and the sink
// will just proceed with no retry at all
desc: "missing max retry",
bucket: localBucket,
overallTimeout: 15 * time.Second,
retryTimeout: 25 * time.Second,
backoffStrategy: defaultBackoffStrategy,
expectedError: nil,
},
{
desc: "missing bucket",
bucket: nil,
overallTimeout: 15 * time.Second,
maxRetry: 0,
retryTimeout: 25 * time.Second,
backoffStrategy: defaultBackoffStrategy,
expectedError: errMissingBucket,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
if tc.allDefault {
_, err := NewSink(tc.bucket)
require.NoError(t, err)
return
}
sink, err := NewSink(tc.bucket, WithOverallTimeout(tc.overallTimeout),
WithMaxRetry(tc.maxRetry),
WithRetryTimeout(tc.retryTimeout),
WithBackoffStrategy(tc.backoffStrategy))
if tc.expectedError != nil {
require.Nil(t, sink)
require.ErrorIs(t, err, tc.expectedError)
} else {
require.NoError(t, err)
}
})
}
}
func TestSink_Upload(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
desc string
prefix string
// filesToUpload is a map of files to be uploaded.
// The key is the file name and value is the file content.
// If content is empty, it means the file does not exist on local filesystem.
filesToUpload map[string]string
expectedSuccessful map[string]string
expectedErrored map[string]error
}{
{
desc: "Upload objects with metadata",
prefix: "jerry",
filesToUpload: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
},
expectedSuccessful: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
},
},
{
desc: "Upload objects but one is missing on local",
prefix: "jerry",
filesToUpload: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
"5126": "", // missing on local
},
expectedSuccessful: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
},
expectedErrored: map[string]error{
"5126": os.ErrNotExist,
},
},
{
desc: "prefix can be empty",
prefix: "",
filesToUpload: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
"5126": "", // missing on local
},
expectedSuccessful: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
},
expectedErrored: map[string]error{
"5126": os.ErrNotExist,
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
sink := setupEmptyLocalBucket(t)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
for fileName, content := range tc.filesToUpload {
if content != "" {
err := os.WriteFile(filepath.Join(localDir, fileName), []byte(content), mode.File)
require.NoError(t, err)
}
objectName := fileName
testMetadata := map[string]string{
"storage-name": "default",
"storage-path": "/var/opt/gitlab/git-data",
"relative-path": "gitlab/gitaly.git",
"partition-id": "partition-1",
}
err := sink.Upload(ctx, filepath.Join(localDir, objectName), tc.prefix, testMetadata)
if err == nil {
objKey := tc.prefix + "/" + objectName
attr, err := sink.bucket.Attributes(ctx, objKey)
require.NoError(t, err)
require.Equal(t, attr.CacheControl, "no-store, no-transform")
require.Equal(t, attr.ContentType, "application/octet-stream")
require.NotNil(t, attr.Metadata, "metadata should be stored with the object")
require.Equal(t, testMetadata, attr.Metadata, "stored metadata should match uploaded metadata for object %s", objectName)
var builder strings.Builder
err = sink.bucket.Download(ctx, objKey, &builder, nil)
require.NoError(t, err)
require.Equal(t, tc.expectedSuccessful[objectName], builder.String())
} else {
require.ErrorIs(t, err, tc.expectedErrored[objectName])
}
}
})
}
t.Run("upload duplicated key whose writer finished later wins", func(t *testing.T) {
ctx := testhelper.Context(t)
localDirLoser := testhelper.TempDir(t)
err := os.WriteFile(filepath.Join(localDirLoser, "i_am_key"), []byte("hello"), mode.File)
require.NoError(t, err)
localDirWinner := testhelper.TempDir(t)
err = os.WriteFile(filepath.Join(localDirWinner, "i_am_key"), []byte("world"), mode.File)
require.NoError(t, err)
sink := setupEmptyLocalBucket(t)
err = sink.Upload(ctx, filepath.Join(localDirLoser, "i_am_key"), "some/prefix", nil)
require.NoError(t, err)
err = sink.Upload(ctx, filepath.Join(localDirWinner, "i_am_key"), "some/prefix", nil)
require.NoError(t, err)
var builder strings.Builder
err = sink.bucket.Download(ctx, "some/prefix/i_am_key", &builder, nil)
require.NoError(t, err)
require.Equal(t, "world", builder.String())
})
t.Run("local file path cannot be empty", func(t *testing.T) {
ctx := testhelper.Context(t)
sink := setupEmptyLocalBucket(t)
defer closeBucket(t, sink)
err := sink.Upload(ctx, "", "some/prefix", nil)
require.Error(t, err)
})
}
func TestSink_Upload_Timeout_Cancellation_And_Retry(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
desc string
maxRetry uint
overallTimeout time.Duration
// operationTimeOut is the timer for each operation e.g. upload, download
operationTimeOut time.Duration
objectName string
simulationData []simulation
expectedError error
}{
{
desc: "upload failed with overall timer overallTimeout no backoffStrategy",
maxRetry: 0,
overallTimeout: 200 * time.Millisecond,
operationTimeOut: 5 * time.Second,
objectName: "overall_timeout_obj_key",
simulationData: []simulation{
{Delay: 1 * time.Second, Err: nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "upload with first two attempts timing out and last backoffStrategy succeeding",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 400 * time.Millisecond,
objectName: "success_with_retry",
simulationData: []simulation{
{5 * time.Second, nil}, // this will timeout
{1 * time.Millisecond, nil}, // this should succeed
},
expectedError: nil,
},
{
desc: "upload failed with operation timer overallTimeout and with backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 100 * time.Millisecond,
objectName: "failed_even_retry",
simulationData: []simulation{
{1 * time.Second, nil},
{1 * time.Second, nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "upload with first two attempts canceled and last backoffStrategy succeeding",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, nil},
},
expectedError: nil,
},
{
desc: "upload failed with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, errSimulationCanceled},
},
expectedError: errSimulationCanceled,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
prefix := "some/prefix"
localBucket := testhelper.TempDir(t)
localBucketURI := fmt.Sprintf("file://%s", localBucket)
bucket, err := blob.OpenBucket(ctx, localBucketURI)
require.NoError(t, err)
objectKey := fmt.Sprintf("%s/%s", prefix, tc.objectName)
simulationData := map[string][]simulation{
objectKey: tc.simulationData,
}
simulatedBucket, err := newSimulationBucket(bucket, simulationData)
require.NoError(t, err)
sink, err := NewSink(simulatedBucket,
WithOverallTimeout(tc.overallTimeout),
WithMaxRetry(tc.maxRetry),
WithRetryTimeout(tc.operationTimeOut),
WithBackoffStrategy(&backoffStrategyInTest))
require.NoError(t, err)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
err = os.WriteFile(filepath.Join(localDir, tc.objectName), []byte("Go long!"), mode.File)
require.NoError(t, err)
err = sink.Upload(ctx, filepath.Join(localDir, tc.objectName), prefix, nil)
if tc.expectedError != nil {
require.ErrorIs(t, err, errSimulationCanceled)
// Add delay here to ensure any pending operations complete
time.Sleep(50 * time.Millisecond)
listRes, err := sink.List(ctx, prefix)
require.NoError(t, err)
require.Empty(t, listRes)
} else {
require.NoError(t, err)
}
})
}
t.Run("upload failed with overall cancel triggered no backoffStrategy", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
prefix := "some/prefix"
objectName := "overall_cancel_called"
localBucket := testhelper.TempDir(t)
localBucketURI := fmt.Sprintf("file://%s", localBucket)
bucket, err := blob.OpenBucket(ctx, localBucketURI)
require.NoError(t, err)
objectKey := fmt.Sprintf("%s/%s", prefix, objectName)
simulation := map[string][]simulation{
objectKey: {
{1 * time.Second, nil},
},
}
simulatedBucket, err := newSimulationBucket(bucket, simulation)
require.NoError(t, err)
ctx, cancel := context.WithCancel(ctx)
// make the overall timer and operation timer are long enough
sink, err := NewSink(simulatedBucket, WithOverallTimeout(100*time.Second), WithNoRetry(),
WithRetryTimeout(10*time.Second), WithBackoffStrategy(&backoffStrategyInTest))
require.NoError(t, err)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
err = os.WriteFile(filepath.Join(localDir, objectName), []byte("Go long!"), mode.File)
require.NoError(t, err)
errCh := make(chan error)
go func() {
errCh <- sink.Upload(ctx, filepath.Join(localDir, objectName), prefix, nil)
}()
// Add a small delay before cancellation to ensure operation has started
time.Sleep(50 * time.Millisecond)
// Trigger the cancellation of the context to stop the ongoing operation.
cancel()
// Add a timeout to prevent test from hanging
select {
case err = <-errCh:
require.ErrorIs(t, err, errSimulationCanceled)
case <-time.After(5 * time.Second):
t.Fatal("Test timed out waiting for cancellation")
}
})
}
func TestSink_Download(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
queryPrefix string
queryObjectNames []string
expectedContent map[string]string
expectedError map[string]gcerrors.ErrorCode
}{
{
desc: "download all objects",
queryPrefix: "jerry",
queryObjectNames: []string{"C-131", "C-137"},
expectedContent: map[string]string{
"C-131": "I am Mr. Frundles",
"C-137": "Why are you here?",
},
},
{
desc: "Download nonexistent objects",
queryPrefix: "jerry",
queryObjectNames: []string{"5126"},
expectedContent: map[string]string{},
expectedError: map[string]gcerrors.ErrorCode{
"5126": gcerrors.NotFound,
},
},
{
desc: "Download nonexistent folder",
queryPrefix: "nonexistent",
queryObjectNames: []string{"C-131"},
expectedContent: map[string]string{},
expectedError: map[string]gcerrors.ErrorCode{
"C-131": gcerrors.NotFound,
},
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
var sink *Sink
prefix := "jerry"
objects := []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
}
sink, _ = setupLocalBucketWithData(t, prefix, objects)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
for _, obj := range tc.queryObjectNames {
objectKey := fmt.Sprintf("%s/%s", tc.queryPrefix, obj)
localFullPath := filepath.Join(localDir, obj)
err := sink.Download(ctx, objectKey, localFullPath)
if err == nil {
buf, err := os.ReadFile(localFullPath)
require.NoError(t, err)
require.Equal(t, tc.expectedContent[obj], string(buf))
} else {
require.Equal(t, tc.expectedError[obj], gcerrors.Code(err))
require.NoFileExists(t, localFullPath)
}
}
})
}
}
func TestSink_Download_Timeout_Cancellation_And_Retry(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
maxRetry uint
overallTimeout time.Duration
// operationTimeOut is the timer for each operation e.g. upload, download
operationTimeOut time.Duration
objectName string
objectContent string
simulationData []simulation
expectedError error
}{
{
desc: "download failed with overall timer overallTimeout no backoffStrategy",
maxRetry: 0,
overallTimeout: 100 * time.Millisecond,
operationTimeOut: 5 * time.Second,
objectName: "overall_timeout_obj_key",
objectContent: "it's about time",
simulationData: []simulation{
{Delay: 10 * time.Second, Err: nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "down success with some timeouts and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 100 * time.Millisecond,
objectName: "success_with_retry",
objectContent: "with some network jitter, we made it",
simulationData: []simulation{
{300 * time.Millisecond, nil}, // this will overallTimeout
{10 * time.Millisecond, nil}, // this should succeed
},
expectedError: nil,
},
{
desc: "download failed with operation overallTimeout and with backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 50 * time.Millisecond,
objectName: "failed_even_retry",
objectContent: "we had a terrible network",
simulationData: []simulation{
{1 * time.Second, nil},
{1 * time.Second, nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "down success with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, nil},
},
expectedError: nil,
},
{
desc: "download failed with simulated cancel and backoffStrategy",
maxRetry: 2,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, errSimulationCanceled},
},
expectedError: errSimulationCanceled,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
prefix := "some/prefix"
objects := []fileBucketData{
{
ObjectName: tc.objectName,
Content: tc.objectContent,
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, tc.objectName)
simulationData := map[string][]simulation{
objectKey: tc.simulationData,
}
simulatedBucket, err := newSimulationBucket(bucket, simulationData)
require.NoError(t, err)
sink, err := NewSink(simulatedBucket,
WithOverallTimeout(tc.overallTimeout),
WithMaxRetry(tc.maxRetry),
WithRetryTimeout(tc.operationTimeOut),
WithBackoffStrategy(&backoffStrategyInTest),
)
require.NoError(t, err)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
localFullPath := filepath.Join(localDir, tc.objectName)
err = sink.Download(ctx, objectKey, localFullPath)
if tc.expectedError != nil {
require.ErrorIs(t, err, tc.expectedError)
require.NoFileExists(t, localFullPath)
} else {
buf, err := os.ReadFile(localFullPath)
require.NoError(t, err)
require.Equal(t, tc.objectContent, string(buf))
}
})
}
t.Run("download failed with overall cancel triggered no backoffStrategy", func(t *testing.T) {
prefix := "some/prefix"
objectName := "overall_cancel_called"
objectContent := "I can't wait forever"
objects := []fileBucketData{
{
ObjectName: objectName,
Content: objectContent,
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, objectName)
simulationData := map[string][]simulation{
objectKey: {
{1 * time.Second, nil},
},
}
simulatedBucket, err := newSimulationBucket(bucket, simulationData)
require.NoError(t, err)
ctx, cancel := context.WithCancel(ctx)
sink, err := NewSink(simulatedBucket, WithOverallTimeout(defaultOverallTimeout),
WithNoRetry(), WithRetryTimeout(defaultRetryTimeout),
WithBackoffStrategy(&backoffStrategyInTest))
require.NoError(t, err)
defer closeBucket(t, sink)
localDir := testhelper.TempDir(t)
localFullPath := filepath.Join(localDir, objectName)
errCh := make(chan error)
go func() {
errCh <- sink.Download(ctx, objectKey, localFullPath)
}()
// Trigger the cancellation of the context to stop the ongoing operation.
cancel()
err = <-errCh
require.ErrorIs(t, err, errSimulationCanceled)
})
}
func TestSink_Delete(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
queryingPrefix string
createBucket bool
bucket string
prefix string
objects []fileBucketData
objectsToDelete []string
objectsLeft []string
expectedRes map[string]gcerrors.ErrorCode
}{
{
desc: "delete all objects",
queryingPrefix: "jerry",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
objectsToDelete: []string{"C-131", "C-137"},
expectedRes: map[string]gcerrors.ErrorCode{},
},
{
desc: "delete some objects",
queryingPrefix: "jerry",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
objectsToDelete: []string{"C-131"},
objectsLeft: []string{"C-137"},
expectedRes: map[string]gcerrors.ErrorCode{},
},
{
desc: "delete more objects than you have",
queryingPrefix: "jerry",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
objectsToDelete: []string{"C-131", "C-137", "5126"},
objectsLeft: []string{},
expectedRes: map[string]gcerrors.ErrorCode{
"jerry/5126": gcerrors.NotFound,
},
},
{
desc: "delete nonexistent prefix",
queryingPrefix: "nonexistent",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
},
objectsToDelete: []string{"C-131"},
objectsLeft: []string{"C-131"},
expectedRes: map[string]gcerrors.ErrorCode{
"nonexistent/C-131": gcerrors.NotFound,
},
},
{
desc: "empty bucket",
queryingPrefix: "i-am-empty",
prefix: "i-am-empty",
objects: []fileBucketData{},
objectsToDelete: []string{"C-131"},
objectsLeft: []string{},
expectedRes: map[string]gcerrors.ErrorCode{
"i-am-empty/C-131": gcerrors.NotFound,
},
},
{
desc: "delete parameter is empty",
queryingPrefix: "jerry",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
objectsToDelete: []string{},
expectedRes: map[string]gcerrors.ErrorCode{},
objectsLeft: []string{"C-131", "C-137"},
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
var sink *Sink
sink, _ = setupLocalBucketWithData(t, tc.prefix, tc.objects)
defer closeBucket(t, sink)
actualResWithErrorCode := make(map[string]gcerrors.ErrorCode)
res := sink.DeleteObjects(ctx, tc.queryingPrefix, tc.objectsToDelete)
for k, v := range res {
actualResWithErrorCode[k] = gcerrors.Code(v)
}
require.Equal(t, tc.expectedRes, actualResWithErrorCode)
actualObjectsLeft, err := sink.List(ctx, tc.prefix)
require.NoError(t, err)
require.ElementsMatch(t, tc.objectsLeft, actualObjectsLeft)
})
}
}
func TestSink_Delete_Timeout_Cancellation_And_Retry(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
maxRetry uint
overallTimeout time.Duration
// operationTimeOut is the timer for each operation e.g. upload, download
operationTimeOut time.Duration
objectName string
simulationData []simulation
expectedError error
}{
{
desc: "delete failed with overall timer overallTimeout no backoffStrategy",
maxRetry: 0,
overallTimeout: 100 * time.Millisecond,
operationTimeOut: 5 * time.Second,
objectName: "overall_timeout_obj_key",
simulationData: []simulation{
{Delay: 10 * time.Second, Err: nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "delete success with some timeouts and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 100 * time.Millisecond,
objectName: "success_with_retry",
simulationData: []simulation{
{300 * time.Millisecond, nil}, // this will overallTimeout
{10 * time.Millisecond, nil}, // this should succeed
},
expectedError: nil,
},
{
desc: "delete failed with operation timeout and with backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 50 * time.Millisecond,
objectName: "failed_even_retry",
simulationData: []simulation{
{1 * time.Second, nil},
{1 * time.Second, nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "delete success with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, nil},
},
expectedError: nil,
},
{
desc: "delete failed with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, errSimulationCanceled},
},
expectedError: errSimulationCanceled,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
prefix := "some/prefix"
objects := []fileBucketData{
{
ObjectName: tc.objectName,
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, tc.objectName)
simulation := map[string][]simulation{
objectKey: tc.simulationData,
}
simulatedBucket, err := newSimulationBucket(bucket, simulation)
require.NoError(t, err)
sink, err := NewSink(simulatedBucket,
WithOverallTimeout(tc.overallTimeout),
WithMaxRetry(tc.maxRetry),
WithRetryTimeout(tc.operationTimeOut),
WithBackoffStrategy(&backoffStrategyInTest),
)
require.NoError(t, err)
defer closeBucket(t, sink)
res := sink.DeleteObjects(ctx, prefix, []string{objectKey})
if tc.expectedError != nil {
require.ErrorIs(t, res[objectKey], tc.expectedError)
} else {
require.NoError(t, res[objectKey])
actualObjectsLeft, err := sink.List(ctx, prefix)
require.NoError(t, err)
require.Empty(t, actualObjectsLeft)
}
})
}
t.Run("download failed with overall cancel triggered no backoffStrategy", func(t *testing.T) {
prefix := "some/prefix"
objectName := "overall_cancel_called"
objectContent := "I can't wait forever"
objects := []fileBucketData{
{
ObjectName: objectName,
Content: objectContent,
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, objectName)
simulation := map[string][]simulation{
objectKey: {
{1 * time.Second, nil},
},
}
simulatedBucket, err := newSimulationBucket(bucket, simulation)
require.NoError(t, err)
ctx, cancel := context.WithCancel(ctx)
sink, err := NewSink(simulatedBucket, WithOverallTimeout(defaultOverallTimeout),
WithNoRetry(), WithRetryTimeout(defaultRetryTimeout),
WithBackoffStrategy(&backoffStrategyInTest))
require.NoError(t, err)
defer closeBucket(t, sink)
errCh := make(chan map[string]error)
go func() {
errCh <- sink.DeleteObjects(ctx, prefix, []string{objectKey})
}()
// Trigger the cancellation of the context to stop the ongoing operation.
cancel()
res := <-errCh
require.ErrorIs(t, res[objectKey], errSimulationCanceled)
})
}
// Test List
func TestSink_List(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
queryingPrefix string
createBucket bool
bucket string
prefix string
objects []fileBucketData
expectedRes []string
expectedErr error
}{
{
desc: "bucket with objects",
queryingPrefix: "jerry",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
expectedRes: []string{"C-131", "C-137"},
expectedErr: nil,
},
{
// The "/" will not affect the result.
desc: "prefix with / ending",
queryingPrefix: "jerry/",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
{
ObjectName: "C-137",
Content: "Why are you here?",
WriterOpt: blob.WriterOptions{
ContentType: "text/html; charset=utf-8",
CacheControl: "no-store, no-transform",
},
},
},
expectedRes: []string{"C-131", "C-137"},
expectedErr: nil,
},
{
desc: "nonexistent prefix in a bucket with objects",
queryingPrefix: "nonexistent",
prefix: "jerry",
objects: []fileBucketData{
{
ObjectName: "C-131",
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
},
expectedRes: []string{},
expectedErr: nil,
},
{
desc: "empty bucket",
queryingPrefix: "i-am-empty",
prefix: "i-am-empty",
objects: []fileBucketData{},
expectedRes: []string{},
expectedErr: nil,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
var sink *Sink
sink, _ = setupLocalBucketWithData(t, tc.prefix, tc.objects)
defer closeBucket(t, sink)
res, err := sink.List(ctx, tc.queryingPrefix)
require.NoError(t, err)
require.Equal(t, tc.expectedRes, res)
})
}
}
func TestSink_List_Timeout_Cancellation_And_Retry(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
maxRetry uint
overallTimeout time.Duration
// operationTimeOut is the timer for each operation e.g. upload, download
operationTimeOut time.Duration
objectName string
simulationData []simulation
expectedError error
}{
{
desc: "list failed with overall timer overallTimeout no backoffStrategy",
maxRetry: 0,
overallTimeout: 100 * time.Millisecond,
operationTimeOut: 5 * time.Second,
objectName: "overall_timeout_obj_key",
simulationData: []simulation{
{Delay: 10 * time.Second, Err: nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "list success with some timeouts and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 100 * time.Millisecond,
objectName: "success_with_retry",
simulationData: []simulation{
{1 * time.Second, nil}, // this will overallTimeout
{1 * time.Millisecond, nil}, // this should succeed
},
expectedError: nil,
},
{
desc: "list failed with operation overallTimeout and with backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 50 * time.Millisecond,
objectName: "failed_even_retry",
simulationData: []simulation{
{1 * time.Second, nil},
{1 * time.Second, nil},
},
expectedError: errSimulationCanceled,
},
{
desc: "list success with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, nil},
},
expectedError: nil,
},
{
desc: "delete failed with simulated cancel and backoffStrategy",
maxRetry: 1,
overallTimeout: 60 * time.Second,
operationTimeOut: 10 * time.Second,
objectName: "success_with_cancel_and_retry",
simulationData: []simulation{
{100 * time.Millisecond, errSimulationCanceled},
{100 * time.Millisecond, errSimulationCanceled},
},
expectedError: errSimulationCanceled,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
prefix := "some/prefix"
objects := []fileBucketData{
{
ObjectName: tc.objectName,
Content: "I am Mr. Frundles",
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, tc.objectName)
simulationData := map[string][]simulation{
objectKey: tc.simulationData,
}
simulatedBucket, err := newSimulationBucket(bucket, simulationData)
require.NoError(t, err)
simulatedBucketPtr, ok := simulatedBucket.(*simulationBucket)
require.True(t, ok)
sink, err := NewSink(simulatedBucket, WithOverallTimeout(tc.overallTimeout),
WithMaxRetry(tc.maxRetry), WithRetryTimeout(tc.operationTimeOut),
WithBackoffStrategy(&backoffStrategyInTest))
require.NoError(t, err)
defer closeBucket(t, sink)
// Notice that we are testing against listHandler instead of the actual List function here.
// The reason is that listHandler can load the intercepted List function, which uses the
// simulation data defined in interceptedList. This approach is valid because listHandler
// serves as the underlying implementation of the List function.
res, err := listHandler(ctx, sink, prefix, simulatedBucketPtr.interceptedList)
if tc.expectedError != nil {
require.ErrorIs(t, err, tc.expectedError)
} else {
require.NoError(t, err)
require.ElementsMatch(t, res, []string{tc.objectName})
}
})
}
t.Run("list failed with overall cancel triggered no backoffStrategy", func(t *testing.T) {
prefix := "some/prefix"
objectName := "overall_cancel_called"
objectContent := "I can't wait forever"
objects := []fileBucketData{
{
ObjectName: objectName,
Content: objectContent,
WriterOpt: blob.WriterOptions{
ContentType: "application/octet-stream",
CacheControl: "no-store, no-transform",
},
},
}
_, bucket := setupLocalBucketWithData(t, prefix, objects)
objectKey := fmt.Sprintf("%s/%s", prefix, objectName)
simulation := map[string][]simulation{
objectKey: {
{1 * time.Second, nil},
},
}
simulatedBucket, err := newSimulationBucket(bucket, simulation)
require.NoError(t, err)
simulatedBucketPtr, ok := simulatedBucket.(*simulationBucket)
require.True(t, ok)
ctx, cancel := context.WithCancel(ctx)
sink, err := NewSink(simulatedBucket)
require.NoError(t, err)
defer closeBucket(t, sink)
errCh := make(chan error)
go func() {
// Notice that we are testing against listHandler instead of the actual List function here.
// The reason is that listHandler can load the intercepted List function, which uses the
// simulation data defined in interceptedList. This approach is valid because listHandler
// serves as the underlying implementation of the List function.
_, err := listHandler(ctx, sink, prefix, simulatedBucketPtr.interceptedList)
errCh <- err
}()
// Trigger the cancellation of the context to stop the ongoing operation.
cancel()
res := <-errCh
require.ErrorIs(t, res, errSimulationCanceled)
})
}
type fileBucketData struct {
ObjectName string
Content string
WriterOpt blob.WriterOptions
}
// setupEmptyLocalBucket initializes an empty Bucket backed by the local file system.
func setupEmptyLocalBucket(t *testing.T) *Sink {
ctx := testhelper.Context(t)
localBucket := testhelper.TempDir(t)
localBucketURI := fmt.Sprintf("file://%s", localBucket)
bucket, err := blob.OpenBucket(ctx, localBucketURI)
require.NoError(t, err)
sink, err := NewSink(bucket, WithOverallTimeout(defaultOverallTimeout),
WithMaxRetry(defaultMaxRetry), WithRetryTimeout(defaultRetryTimeout),
WithBackoffStrategy(&constantBackoff{}))
require.NoError(t, err)
return sink
}
// setupLocalBucketWithData initializes a Bucket backed by the local file system with data.
func setupLocalBucketWithData(t *testing.T, prefix string, objectsToUpload []fileBucketData) (*Sink, *blob.Bucket) {
ctx := testhelper.Context(t)
bucket := testhelper.TempDir(t)
localBucketURI := fmt.Sprintf("file://%s", bucket)
localBucket, err := blob.OpenBucket(ctx, localBucketURI)
require.NoError(t, err)
sink, err := NewSink(localBucket, WithOverallTimeout(defaultOverallTimeout),
WithMaxRetry(defaultMaxRetry), WithRetryTimeout(defaultRetryTimeout),
WithBackoffStrategy(&constantBackoff{}))
require.NoError(t, err)
for _, obj := range objectsToUpload {
objectKey := filepath.Join(prefix, obj.ObjectName)
content := strings.NewReader(obj.Content)
require.NoError(t, err)
err = sink.bucket.Upload(ctx, objectKey, content, &obj.WriterOpt)
require.NoError(t, err)
}
return sink, localBucket
}
func closeBucket(t *testing.T, sink *Sink) {
err := sink.bucket.Close()
require.NoError(t, err)
}
// constantBackoff always returns constant backoff time intervals regardless of the backoffStrategy attempts.
type constantBackoff struct{}
func (c *constantBackoff) Backoff(uint) time.Duration {
return time.Duration(100) * time.Millisecond
}