Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

11.5 KB
Raw Blame
package test

import (
	"bytes"
	"context"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/usememos/memos/internal/testutil/fakes3"
	apipb "github.com/usememos/memos/proto/gen/api"
	storepb "github.com/usememos/memos/proto/gen/store"
	"github.com/usememos/memos/provider/storage/s3"
	"github.com/usememos/memos/server/api"
	"github.com/usememos/memos/store"
)

func TestS3AttachmentLifecycleAcrossStorageChange(t *testing.T) {
	fake := fakes3.New(t, "attachments-old", "attachments-new")
	runS3AttachmentLifecycleAcrossStorageChange(t, fake)
}

func TestUploadAttachmentS3(t *testing.T) {
	ts := NewTestService(t)
	defer ts.Cleanup()
	defer ts.Service.CloseUploads()
	fake := fakes3.New(t, "uploads")
	ctx := context.Background()
	user, err := ts.CreateRegularUser(ctx, "chunk-uploader")
	require.NoError(t, err)
	ctx = ts.CreateUserContext(ctx, user.ID)
	storage := fakeStorage("upload-s3", "Uploads", fake.Config("uploads"))
	upsertS3StorageSetting(ctx, t, ts, storage.Id, storage)
	chunk := bytes.Repeat([]byte("x"), 2<<20)
	initial, err := ts.Service.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: &apipb.UploadAttachmentRequest_Spec{Spec: &apipb.UploadAttachmentSpec{
		Attachment: &apipb.Attachment{Filename: "large.bin", Type: "application/octet-stream"}, TotalSize: int64(3 * len(chunk)),
	}}})
	require.NoError(t, err)
	var result *apipb.UploadAttachmentResponse
	for i := range 3 {
		result, err = ts.Service.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{
			Upload: &apipb.UploadAttachmentRequest_UploadId{UploadId: initial.UploadId}, WriteOffset: int64(i * len(chunk)), Data: chunk, FinishWrite: i == 2,
		})
		require.NoError(t, err)
	}
	uid, err := api.ExtractAttachmentUIDFromName(result.Attachment.Name)
	require.NoError(t, err)
	row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
	require.NoError(t, err)
	require.Empty(t, row.Blob)
	require.Equal(t, storepb.AttachmentStorageType_S3, row.StorageType)
	content, err := fake.GetObject("uploads", row.Payload.GetS3Object().GetKey())
	require.NoError(t, err)
	require.Equal(t, bytes.Repeat(chunk, 3), content)
}

func runS3AttachmentLifecycleAcrossStorageChange(t *testing.T, objectStore *fakes3.Server) {
	t.Helper()
	ctx := context.Background()
	ts := NewTestService(t)
	defer ts.Cleanup()

	user, err := ts.CreateRegularUser(ctx, "s3-attachment-user")
	require.NoError(t, err)
	userCtx := ts.CreateUserContext(ctx, user.ID)

	oldStorage := fakeStorage("s3-old", "Old S3", objectStore.Config("attachments-old"))
	upsertS3StorageSetting(ctx, t, ts, oldStorage.Id, oldStorage)

	firstContent := []byte("first attachment in old storage")
	first, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{
		Attachment: &apipb.Attachment{
			Filename: "first.txt",
			Type:     "text/plain",
			Content:  firstContent,
		},
	})
	require.NoError(t, err)

	firstUID, err := api.ExtractAttachmentUIDFromName(first.Name)
	require.NoError(t, err)
	storedFirst, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &firstUID})
	require.NoError(t, err)
	require.NotNil(t, storedFirst)
	require.Equal(t, storepb.AttachmentStorageType_S3, storedFirst.StorageType)
	require.Empty(t, storedFirst.Blob)
	require.Empty(t, storedFirst.Reference, "S3 attachments must not store a reference URL")
	require.Equal(t, oldStorage.Id, storedFirst.Payload.GetS3Object().GetStorageId())
	require.Nil(t, storedFirst.Payload.GetS3Object().GetS3Config(), "new attachments must not embed S3 credentials")

	oldKey := storedFirst.Payload.GetS3Object().GetKey()
	storedInOldBucket, err := objectStore.GetObject("attachments-old", oldKey)
	require.NoError(t, err)
	require.Equal(t, firstContent, storedInOldBucket)
	firstDownloaded, err := ts.Service.GetAttachmentBlob(ctx, storedFirst)
	require.NoError(t, err)
	require.Equal(t, firstContent, firstDownloaded)
	firstMetadata, err := ts.Service.GetAttachment(userCtx, &apipb.GetAttachmentRequest{Name: first.Name})
	require.NoError(t, err)
	require.Equal(t, first.Name, firstMetadata.Name)
	require.Empty(t, firstMetadata.ExternalLink)

	newStorage := fakeStorage("s3-new", "New S3", objectStore.Config("attachments-new"))
	upsertS3StorageSetting(ctx, t, ts, newStorage.Id, oldStorage, newStorage)

	secondContent := []byte("second attachment in new storage")
	second, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{
		Attachment: &apipb.Attachment{
			Filename: "second.txt",
			Type:     "text/plain",
			Content:  secondContent,
		},
	})
	require.NoError(t, err)
	secondUID, err := api.ExtractAttachmentUIDFromName(second.Name)
	require.NoError(t, err)
	storedSecond, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &secondUID})
	require.NoError(t, err)
	require.Equal(t, newStorage.Id, storedSecond.Payload.GetS3Object().GetStorageId())
	newKey := storedSecond.Payload.GetS3Object().GetKey()
	storedInNewBucket, err := objectStore.GetObject("attachments-new", newKey)
	require.NoError(t, err)
	require.Equal(t, secondContent, storedInNewBucket)

	// Changing the default storage only affects new uploads. The first attachment
	// must continue to resolve its original storage by ID.
	firstDownloaded, err = ts.Service.GetAttachmentBlob(ctx, storedFirst)
	require.NoError(t, err)
	require.Equal(t, firstContent, firstDownloaded)
	secondDownloaded, err := ts.Service.GetAttachmentBlob(ctx, storedSecond)
	require.NoError(t, err)
	require.Equal(t, secondContent, secondDownloaded)

	_, err = ts.Service.DeleteAttachment(userCtx, &apipb.DeleteAttachmentRequest{Name: first.Name})
	require.NoError(t, err)
	_, err = objectStore.GetObject("attachments-old", oldKey)
	require.Error(t, err, "deleting an attachment must delete its S3 object")
	remaining, err := objectStore.GetObject("attachments-new", newKey)
	require.NoError(t, err)
	require.Equal(t, secondContent, remaining)
}

func TestGetLegacyS3AttachmentBlob(t *testing.T) {
	ctx := context.Background()
	fake := fakes3.New(t, "legacy-fallback", "legacy-current")
	ts := NewTestService(t)
	defer ts.Cleanup()

	user, err := ts.CreateRegularUser(ctx, "legacy-s3-user")
	require.NoError(t, err)
	userCtx := ts.CreateUserContext(ctx, user.ID)

	t.Run("falls back to embedded config when storage is not registered", func(t *testing.T) {
		legacyConfig := fake.Config("legacy-fallback")
		driver, err := s3.NewDriver(ctx, legacyConfig)
		require.NoError(t, err)
		content := []byte("attachment created before storage IDs")
		key, err := driver.UploadObject(ctx, "legacy/fallback.txt", "text/plain", bytes.NewReader(content))
		require.NoError(t, err)

		legacyAttachment, err := ts.Store.CreateAttachment(ctx, &store.Attachment{
			UID:         "legacy-s3-fallback",
			CreatorID:   user.ID,
			Filename:    "fallback.txt",
			Type:        "text/plain",
			Size:        int64(len(content)),
			StorageType: storepb.AttachmentStorageType_S3,
			Payload: &storepb.AttachmentPayload{
				Payload: &storepb.AttachmentPayload_S3Object_{
					S3Object: &storepb.AttachmentPayload_S3Object{
						S3Config: legacyConfig,
						Key:      key,
					},
				},
			},
		})
		require.NoError(t, err)

		downloaded, err := ts.Service.GetAttachmentBlob(ctx, legacyAttachment)
		require.NoError(t, err)
		require.Equal(t, content, downloaded)
	})

	t.Run("uses current storage options for matching namespace", func(t *testing.T) {
		currentConfig := fake.Config("legacy-current")
		driver, err := s3.NewDriver(ctx, currentConfig)
		require.NoError(t, err)
		content := []byte("legacy attachment using rotated storage config")
		key, err := driver.UploadObject(ctx, "legacy/current.txt", "text/plain", bytes.NewReader(content))
		require.NoError(t, err)
		configuredStorage := fakeStorage("s3-current", "Current S3", currentConfig)
		upsertS3StorageSetting(ctx, t, ts, configuredStorage.Id, configuredStorage)

		// Virtual-hosted addressing cannot resolve against httptest's IP endpoint.
		// Retrieval only succeeds if the legacy namespace is matched to the named
		// storage and its current path-style option is used.
		legacyConfig := &storepb.StorageS3Config{
			AccessKeyId:     "old-access-key",
			AccessKeySecret: "old-secret-key",
			Endpoint:        currentConfig.Endpoint + "/",
			Region:          currentConfig.Region,
			Bucket:          currentConfig.Bucket,
			UsePathStyle:    false,
		}
		legacyAttachment, err := ts.Store.CreateAttachment(ctx, &store.Attachment{
			UID:         "legacy-s3-current",
			CreatorID:   user.ID,
			Filename:    "current.txt",
			Type:        "text/plain",
			Size:        int64(len(content)),
			StorageType: storepb.AttachmentStorageType_S3,
			Payload: &storepb.AttachmentPayload{
				Payload: &storepb.AttachmentPayload_S3Object_{
					S3Object: &storepb.AttachmentPayload_S3Object{
						S3Config: legacyConfig,
						Key:      key,
					},
				},
			},
		})
		require.NoError(t, err)

		downloaded, err := ts.Service.GetAttachmentBlob(ctx, legacyAttachment)
		require.NoError(t, err)
		require.Equal(t, content, downloaded)

		_, err = ts.Service.DeleteAttachment(userCtx, &apipb.DeleteAttachmentRequest{Name: api.AttachmentNamePrefix + legacyAttachment.UID})
		require.NoError(t, err)
		_, err = fake.GetObject(currentConfig.Bucket, key)
		require.Error(t, err, "deleting a legacy attachment must use the current matching storage configuration")
	})
}

func fakeStorage(id, name string, config *storepb.StorageS3Config) *storepb.Storage {
	return &storepb.Storage{
		Id:     id,
		Name:   name,
		Type:   storepb.StorageType_STORAGE_TYPE_S3,
		Config: &storepb.Storage_S3Config{S3Config: config},
	}
}

func upsertS3StorageSetting(ctx context.Context, t testing.TB, ts *TestService, defaultStorageID string, storages ...*storepb.Storage) {
	t.Helper()
	_, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
		Key: storepb.InstanceSettingKey_STORAGE,
		Value: &storepb.InstanceSetting_StorageSetting{
			StorageSetting: &storepb.InstanceStorageSetting{
				FilepathTemplate: "attachments/{uuid}_{filename}",
				Storages:         storages,
				DefaultStorageId: defaultStorageID,
			},
		},
	})
	require.NoError(t, err)
}

// Two uploads with the same filename under a path template that carries no
// random component must not overwrite each other's object.
func TestS3ObjectKeysStayUniqueWithoutUUIDTemplate(t *testing.T) {
	ts := NewTestService(t)
	defer ts.Cleanup()
	fake := fakes3.New(t, "flat")
	ctx := context.Background()
	storage := fakeStorage("flat-s3", "Flat", fake.Config("flat"))
	_, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
		Key: storepb.InstanceSettingKey_STORAGE,
		Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{
			FilepathTemplate: "files/{filename}", Storages: []*storepb.Storage{storage}, DefaultStorageId: storage.Id,
		}},
	})
	require.NoError(t, err)

	keys := map[string]bool{}
	for i, username := range []string{"first-user", "second-user"} {
		user, err := ts.CreateRegularUser(ctx, username)
		require.NoError(t, err)
		userCtx := ts.CreateUserContext(ctx, user.ID)
		content := []byte("content of user " + username)
		created, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{Attachment: &apipb.Attachment{
			Filename: "notes.txt", Type: "text/plain", Content: content,
		}})
		require.NoError(t, err)
		uid, err := api.ExtractAttachmentUIDFromName(created.Name)
		require.NoError(t, err)
		row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
		require.NoError(t, err)
		key := row.Payload.GetS3Object().GetKey()
		require.False(t, keys[key], "upload %d reused key %q", i, key)
		keys[key] = true
		stored, err := fake.GetObject("flat", key)
		require.NoError(t, err)
		require.Equal(t, content, stored)
	}
}