Git designed for AI agents.
package backup
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"testing"
"testing/fstest"
"time"
"cloud.google.com/go/storage"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/stretchr/testify/require"
_ "gocloud.dev/blob/memblob"
)
func TestResolveSink(t *testing.T) {
ctx := testhelper.Context(t)
isStorageServiceSink := func(bucketTy interface{}) func(t *testing.T, sink *Sink) {
return func(t *testing.T, sink *Sink) {
t.Helper()
require.True(t, sink.bucket.As(bucketTy))
}
}
tmpDir := testhelper.TempDir(t)
gsCreds := filepath.Join(tmpDir, "gs.creds")
var (
azureBucket *container.Client
gcsBucket *storage.Client
s3Bucket *s3.Client
fileblobBucket os.FileInfo
)
// A syntactically valid but fake service account: the sink only needs a
// credentials file it can parse to build the GCS client; nothing is signed.
gsCredsJSON, err := json.Marshal(map[string]string{
"type": "service_account",
"project_id": "gitai-test",
"private_key_id": "0000000000000000000000000000000000000000",
"private_key": "not-a-real-key",
"client_email": "[email protected]",
"client_id": "000000000000000000000",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
})
require.NoError(t, err)
require.NoError(t, os.WriteFile(gsCreds, gsCredsJSON, mode.File))
for _, tc := range []struct {
desc string
envs map[string]string
path string
verify func(t *testing.T, sink *Sink)
errMsg string
}{
{
desc: "AWS S3",
envs: map[string]string{
"AWS_ACCESS_KEY_ID": "test",
"AWS_SECRET_ACCESS_KEY": "test",
"AWS_REGION": "us-east-1",
},
path: "s3://bucket",
verify: isStorageServiceSink(&s3Bucket),
},
{
desc: "Google Cloud Storage",
envs: map[string]string{
"GOOGLE_APPLICATION_CREDENTIALS": gsCreds,
},
path: "blob+gs://bucket",
verify: isStorageServiceSink(&gcsBucket),
},
{
desc: "Azure Cloud File Storage",
envs: map[string]string{
"AZURE_STORAGE_ACCOUNT": "test",
"AZURE_STORAGE_KEY": "test",
"AZURE_STORAGE_SAS_TOKEN": "test",
},
path: "blob+bucket+azblob://bucket",
verify: isStorageServiceSink(&azureBucket),
},
{
desc: "Fileblob",
path: "file://" + tmpDir,
verify: isStorageServiceSink(&fileblobBucket),
},
{
desc: "Non-existent Fileblob",
path: "file://" + filepath.Join(tmpDir, "some-new-dir"),
verify: isStorageServiceSink(&fileblobBucket),
},
{
desc: "Unspecified scheme uses fileblob",
path: tmpDir,
verify: isStorageServiceSink(&fileblobBucket),
},
{
desc: "undefined",
path: "some:invalid:path\x00",
errMsg: `parse "some:invalid:path\x00": net/url: invalid control character in URL`,
},
{
desc: "unrecognized scheme",
path: "minio://bucket",
errMsg: `unsupported sink URI scheme: "minio"`,
},
} {
t.Run(tc.desc, func(t *testing.T) {
for k, v := range tc.envs {
t.Setenv(k, v)
}
sink, err := ResolveSink(ctx, tc.path)
if tc.errMsg != "" {
require.EqualError(t, err, tc.errMsg)
return
}
require.NoError(t, err)
defer testhelper.MustClose(t, sink)
tc.verify(t, sink)
})
}
}
func TestFileBlobSink(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
tmpDir := testhelper.TempDir(t)
tmpPath := filepath.Join(tmpDir, "another-dir")
sink, err := ResolveSink(ctx, fmt.Sprintf("file://%s", tmpPath))
defer testhelper.MustClose(t, sink)
require.NoError(t, err)
info, err := os.Stat(tmpPath)
require.NoError(t, err)
require.Equal(t, mode.Directory, info.Mode())
}
func TestStorageServiceSink(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
sss, err := ResolveSink(ctx, "mem://test_bucket")
require.NoError(t, err)
defer func() { require.NoError(t, sss.Close()) }()
t.Run("write and retrieve", func(t *testing.T) {
const relativePath = "path/to/data"
data := []byte("test")
w, err := sss.GetWriter(ctx, relativePath)
require.NoError(t, err)
_, err = io.Copy(w, bytes.NewReader(data))
require.NoError(t, err)
require.NoError(t, w.Close())
reader, err := sss.GetReader(ctx, relativePath)
require.NoError(t, err)
defer func() { require.NoError(t, reader.Close()) }()
retrieved, err := io.ReadAll(reader)
require.NoError(t, err)
require.Equal(t, data, retrieved)
})
t.Run("not existing path", func(t *testing.T) {
reader, err := sss.GetReader(ctx, "not-existing")
require.Equal(t, fmt.Errorf(`sink: new reader for "not-existing": %w`, ErrDoesntExist), err)
require.Nil(t, reader)
})
}
func TestStorageServiceSink_SignedURL_notImplemented(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
tmpDir := testhelper.TempDir(t)
for _, tc := range []struct {
desc string
bucketURL string
}{
{
desc: "memory bucket",
bucketURL: "mem://test_bucket",
},
{
desc: "fs bucket",
bucketURL: tmpDir,
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
sss, err := ResolveSink(ctx, tc.bucketURL)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, sss.Close()) })
const relativePath = "path/to/data"
data := []byte("test")
w, err := sss.GetWriter(ctx, relativePath)
require.NoError(t, err)
_, err = io.Copy(w, bytes.NewReader(data))
require.NoError(t, err)
require.NoError(t, w.Close())
_, err = sss.SignedURL(ctx, relativePath, 10*time.Minute)
require.Error(t, err)
require.Contains(t, err.Error(), "Unimplemented")
})
}
}
func TestSink_List(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
rootPath := filepath.Join(t.TempDir(), "sink")
expectedObjects := []string{
"a/b/apple",
"a/b/banana",
"a/b/c/carrot",
"a/b/pineapple",
}
fs := fstest.MapFS{
".": {Mode: mode.Directory},
"a": {Mode: mode.Directory},
"a/b": {Mode: mode.Directory},
"a/b/c": {Mode: mode.Directory},
}
for _, path := range expectedObjects {
fs[path] = &fstest.MapFile{Mode: mode.File, Data: []byte(path)}
}
testhelper.CreateFS(t, rootPath, fs)
sink, err := ResolveSink(ctx, rootPath)
require.NoError(t, err)
defer testhelper.MustClose(t, sink)
var objects []string
it := sink.List("a/b/")
for it.Next(ctx) {
objects = append(objects, it.Path())
}
require.NoError(t, it.Err())
require.Equal(t, expectedObjects, objects)
}
func TestWithS3DefaultChecksumCalculation(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
desc string
inputURL string
expected string
}{
{
desc: "no change for AWS S3 without endpoint",
inputURL: "s3://bucket",
expected: "s3://bucket",
},
{
desc: "no change for AWS S3 with region only",
inputURL: "s3://bucket?region=us-east-1",
expected: "s3://bucket?region=us-east-1",
},
{
desc: "adds default when custom endpoint is configured",
inputURL: "s3://bucket?endpoint=minio.example.com",
expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
},
{
desc: "adds default when custom endpoint with other params",
inputURL: "s3://bucket?region=us-east-1&endpoint=minio.example.com",
expected: "s3://bucket?endpoint=minio.example.com®ion=us-east-1&request_checksum_calculation=when_required",
},
{
desc: "preserves user-specified when_supported with endpoint",
inputURL: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_supported",
expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_supported",
},
{
desc: "preserves user-specified when_required with endpoint",
inputURL: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
},
{
desc: "preserves user-specified checksum for AWS S3 without endpoint",
inputURL: "s3://bucket?request_checksum_calculation=when_supported",
expected: "s3://bucket?request_checksum_calculation=when_supported",
},
{
desc: "handles bucket with path and custom endpoint",
inputURL: "s3://bucket/prefix/path?endpoint=minio.example.com",
expected: "s3://bucket/prefix/path?endpoint=minio.example.com&request_checksum_calculation=when_required",
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
parsed, err := url.Parse(tc.inputURL)
require.NoError(t, err)
result := withS3DefaultChecksumCalculation(parsed)
// Parse both URLs to compare query params regardless of order
resultParsed, err := url.Parse(result)
require.NoError(t, err)
expectedParsed, err := url.Parse(tc.expected)
require.NoError(t, err)
require.Equal(t, expectedParsed.Scheme, resultParsed.Scheme)
require.Equal(t, expectedParsed.Host, resultParsed.Host)
require.Equal(t, expectedParsed.Path, resultParsed.Path)
require.Equal(t, expectedParsed.Query(), resultParsed.Query())
})
}
}