Skip to content

Git designed for AI agents.

9.1 KB
Raw Blame
package backup

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"testing"
	"testing/fstest"
	"time"

	"cloud.google.com/go/storage"
	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/stretchr/testify/require"

	_ "gocloud.dev/blob/memblob"
)

func TestResolveSink(t *testing.T) {
	ctx := testhelper.Context(t)

	isStorageServiceSink := func(bucketTy interface{}) func(t *testing.T, sink *Sink) {
		return func(t *testing.T, sink *Sink) {
			t.Helper()

			require.True(t, sink.bucket.As(bucketTy))
		}
	}

	tmpDir := testhelper.TempDir(t)
	gsCreds := filepath.Join(tmpDir, "gs.creds")

	var (
		azureBucket    *container.Client
		gcsBucket      *storage.Client
		s3Bucket       *s3.Client
		fileblobBucket os.FileInfo
	)

	// A syntactically valid but fake service account: the sink only needs a
	// credentials file it can parse to build the GCS client; nothing is signed.
	gsCredsJSON, err := json.Marshal(map[string]string{
		"type":           "service_account",
		"project_id":     "gitai-test",
		"private_key_id": "0000000000000000000000000000000000000000",
		"private_key":    "not-a-real-key",
		"client_email":   "[email protected]",
		"client_id":      "000000000000000000000",
		"auth_uri":       "https://accounts.google.com/o/oauth2/auth",
		"token_uri":      "https://accounts.google.com/o/oauth2/token",
	})
	require.NoError(t, err)
	require.NoError(t, os.WriteFile(gsCreds, gsCredsJSON, mode.File))

	for _, tc := range []struct {
		desc   string
		envs   map[string]string
		path   string
		verify func(t *testing.T, sink *Sink)
		errMsg string
	}{
		{
			desc: "AWS S3",
			envs: map[string]string{
				"AWS_ACCESS_KEY_ID":     "test",
				"AWS_SECRET_ACCESS_KEY": "test",
				"AWS_REGION":            "us-east-1",
			},
			path:   "s3://bucket",
			verify: isStorageServiceSink(&s3Bucket),
		},
		{
			desc: "Google Cloud Storage",
			envs: map[string]string{
				"GOOGLE_APPLICATION_CREDENTIALS": gsCreds,
			},
			path:   "blob+gs://bucket",
			verify: isStorageServiceSink(&gcsBucket),
		},
		{
			desc: "Azure Cloud File Storage",
			envs: map[string]string{
				"AZURE_STORAGE_ACCOUNT":   "test",
				"AZURE_STORAGE_KEY":       "test",
				"AZURE_STORAGE_SAS_TOKEN": "test",
			},
			path:   "blob+bucket+azblob://bucket",
			verify: isStorageServiceSink(&azureBucket),
		},
		{
			desc:   "Fileblob",
			path:   "file://" + tmpDir,
			verify: isStorageServiceSink(&fileblobBucket),
		},
		{
			desc:   "Non-existent Fileblob",
			path:   "file://" + filepath.Join(tmpDir, "some-new-dir"),
			verify: isStorageServiceSink(&fileblobBucket),
		},
		{
			desc:   "Unspecified scheme uses fileblob",
			path:   tmpDir,
			verify: isStorageServiceSink(&fileblobBucket),
		},
		{
			desc:   "undefined",
			path:   "some:invalid:path\x00",
			errMsg: `parse "some:invalid:path\x00": net/url: invalid control character in URL`,
		},
		{
			desc:   "unrecognized scheme",
			path:   "minio://bucket",
			errMsg: `unsupported sink URI scheme: "minio"`,
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			for k, v := range tc.envs {
				t.Setenv(k, v)
			}

			sink, err := ResolveSink(ctx, tc.path)
			if tc.errMsg != "" {
				require.EqualError(t, err, tc.errMsg)
				return
			}

			require.NoError(t, err)
			defer testhelper.MustClose(t, sink)

			tc.verify(t, sink)
		})
	}
}

func TestFileBlobSink(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	tmpDir := testhelper.TempDir(t)
	tmpPath := filepath.Join(tmpDir, "another-dir")

	sink, err := ResolveSink(ctx, fmt.Sprintf("file://%s", tmpPath))
	defer testhelper.MustClose(t, sink)
	require.NoError(t, err)

	info, err := os.Stat(tmpPath)
	require.NoError(t, err)
	require.Equal(t, mode.Directory, info.Mode())
}

func TestStorageServiceSink(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	sss, err := ResolveSink(ctx, "mem://test_bucket")
	require.NoError(t, err)
	defer func() { require.NoError(t, sss.Close()) }()

	t.Run("write and retrieve", func(t *testing.T) {
		const relativePath = "path/to/data"

		data := []byte("test")

		w, err := sss.GetWriter(ctx, relativePath)
		require.NoError(t, err)

		_, err = io.Copy(w, bytes.NewReader(data))
		require.NoError(t, err)

		require.NoError(t, w.Close())

		reader, err := sss.GetReader(ctx, relativePath)
		require.NoError(t, err)
		defer func() { require.NoError(t, reader.Close()) }()

		retrieved, err := io.ReadAll(reader)
		require.NoError(t, err)
		require.Equal(t, data, retrieved)
	})

	t.Run("not existing path", func(t *testing.T) {
		reader, err := sss.GetReader(ctx, "not-existing")
		require.Equal(t, fmt.Errorf(`sink: new reader for "not-existing": %w`, ErrDoesntExist), err)
		require.Nil(t, reader)
	})
}

func TestStorageServiceSink_SignedURL_notImplemented(t *testing.T) {
	t.Parallel()

	ctx := testhelper.Context(t)
	tmpDir := testhelper.TempDir(t)

	for _, tc := range []struct {
		desc      string
		bucketURL string
	}{
		{
			desc:      "memory bucket",
			bucketURL: "mem://test_bucket",
		},
		{
			desc:      "fs bucket",
			bucketURL: tmpDir,
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			t.Parallel()

			sss, err := ResolveSink(ctx, tc.bucketURL)
			require.NoError(t, err)
			t.Cleanup(func() { require.NoError(t, sss.Close()) })

			const relativePath = "path/to/data"

			data := []byte("test")

			w, err := sss.GetWriter(ctx, relativePath)
			require.NoError(t, err)

			_, err = io.Copy(w, bytes.NewReader(data))
			require.NoError(t, err)

			require.NoError(t, w.Close())

			_, err = sss.SignedURL(ctx, relativePath, 10*time.Minute)
			require.Error(t, err)
			require.Contains(t, err.Error(), "Unimplemented")
		})
	}
}

func TestSink_List(t *testing.T) {
	t.Parallel()

	ctx := testhelper.Context(t)
	rootPath := filepath.Join(t.TempDir(), "sink")

	expectedObjects := []string{
		"a/b/apple",
		"a/b/banana",
		"a/b/c/carrot",
		"a/b/pineapple",
	}

	fs := fstest.MapFS{
		".":     {Mode: mode.Directory},
		"a":     {Mode: mode.Directory},
		"a/b":   {Mode: mode.Directory},
		"a/b/c": {Mode: mode.Directory},
	}

	for _, path := range expectedObjects {
		fs[path] = &fstest.MapFile{Mode: mode.File, Data: []byte(path)}
	}

	testhelper.CreateFS(t, rootPath, fs)

	sink, err := ResolveSink(ctx, rootPath)
	require.NoError(t, err)
	defer testhelper.MustClose(t, sink)

	var objects []string
	it := sink.List("a/b/")
	for it.Next(ctx) {
		objects = append(objects, it.Path())
	}
	require.NoError(t, it.Err())
	require.Equal(t, expectedObjects, objects)
}

func TestWithS3DefaultChecksumCalculation(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc     string
		inputURL string
		expected string
	}{
		{
			desc:     "no change for AWS S3 without endpoint",
			inputURL: "s3://bucket",
			expected: "s3://bucket",
		},
		{
			desc:     "no change for AWS S3 with region only",
			inputURL: "s3://bucket?region=us-east-1",
			expected: "s3://bucket?region=us-east-1",
		},
		{
			desc:     "adds default when custom endpoint is configured",
			inputURL: "s3://bucket?endpoint=minio.example.com",
			expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
		},
		{
			desc:     "adds default when custom endpoint with other params",
			inputURL: "s3://bucket?region=us-east-1&endpoint=minio.example.com",
			expected: "s3://bucket?endpoint=minio.example.com&region=us-east-1&request_checksum_calculation=when_required",
		},
		{
			desc:     "preserves user-specified when_supported with endpoint",
			inputURL: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_supported",
			expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_supported",
		},
		{
			desc:     "preserves user-specified when_required with endpoint",
			inputURL: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
			expected: "s3://bucket?endpoint=minio.example.com&request_checksum_calculation=when_required",
		},
		{
			desc:     "preserves user-specified checksum for AWS S3 without endpoint",
			inputURL: "s3://bucket?request_checksum_calculation=when_supported",
			expected: "s3://bucket?request_checksum_calculation=when_supported",
		},
		{
			desc:     "handles bucket with path and custom endpoint",
			inputURL: "s3://bucket/prefix/path?endpoint=minio.example.com",
			expected: "s3://bucket/prefix/path?endpoint=minio.example.com&request_checksum_calculation=when_required",
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			t.Parallel()

			parsed, err := url.Parse(tc.inputURL)
			require.NoError(t, err)

			result := withS3DefaultChecksumCalculation(parsed)

			// Parse both URLs to compare query params regardless of order
			resultParsed, err := url.Parse(result)
			require.NoError(t, err)
			expectedParsed, err := url.Parse(tc.expected)
			require.NoError(t, err)

			require.Equal(t, expectedParsed.Scheme, resultParsed.Scheme)
			require.Equal(t, expectedParsed.Host, resultParsed.Host)
			require.Equal(t, expectedParsed.Path, resultParsed.Path)
			require.Equal(t, expectedParsed.Query(), resultParsed.Query())
		})
	}
}