Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
package test
import (
"bytes"
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/testutil/fakes3"
apipb "github.com/usememos/memos/proto/gen/api"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/provider/storage/s3"
"github.com/usememos/memos/server/api"
"github.com/usememos/memos/store"
)
func TestS3AttachmentLifecycleAcrossStorageChange(t *testing.T) {
fake := fakes3.New(t, "attachments-old", "attachments-new")
runS3AttachmentLifecycleAcrossStorageChange(t, fake)
}
func TestUploadAttachmentS3(t *testing.T) {
ts := NewTestService(t)
defer ts.Cleanup()
defer ts.Service.CloseUploads()
fake := fakes3.New(t, "uploads")
ctx := context.Background()
user, err := ts.CreateRegularUser(ctx, "chunk-uploader")
require.NoError(t, err)
ctx = ts.CreateUserContext(ctx, user.ID)
storage := fakeStorage("upload-s3", "Uploads", fake.Config("uploads"))
upsertS3StorageSetting(ctx, t, ts, storage.Id, storage)
chunk := bytes.Repeat([]byte("x"), 2<<20)
initial, err := ts.Service.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: &apipb.UploadAttachmentRequest_Spec{Spec: &apipb.UploadAttachmentSpec{
Attachment: &apipb.Attachment{Filename: "large.bin", Type: "application/octet-stream"}, TotalSize: int64(3 * len(chunk)),
}}})
require.NoError(t, err)
var result *apipb.UploadAttachmentResponse
for i := range 3 {
result, err = ts.Service.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{
Upload: &apipb.UploadAttachmentRequest_UploadId{UploadId: initial.UploadId}, WriteOffset: int64(i * len(chunk)), Data: chunk, FinishWrite: i == 2,
})
require.NoError(t, err)
}
uid, err := api.ExtractAttachmentUIDFromName(result.Attachment.Name)
require.NoError(t, err)
row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
require.NoError(t, err)
require.Empty(t, row.Blob)
require.Equal(t, storepb.AttachmentStorageType_S3, row.StorageType)
content, err := fake.GetObject("uploads", row.Payload.GetS3Object().GetKey())
require.NoError(t, err)
require.Equal(t, bytes.Repeat(chunk, 3), content)
}
func runS3AttachmentLifecycleAcrossStorageChange(t *testing.T, objectStore *fakes3.Server) {
t.Helper()
ctx := context.Background()
ts := NewTestService(t)
defer ts.Cleanup()
user, err := ts.CreateRegularUser(ctx, "s3-attachment-user")
require.NoError(t, err)
userCtx := ts.CreateUserContext(ctx, user.ID)
oldStorage := fakeStorage("s3-old", "Old S3", objectStore.Config("attachments-old"))
upsertS3StorageSetting(ctx, t, ts, oldStorage.Id, oldStorage)
firstContent := []byte("first attachment in old storage")
first, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{
Attachment: &apipb.Attachment{
Filename: "first.txt",
Type: "text/plain",
Content: firstContent,
},
})
require.NoError(t, err)
firstUID, err := api.ExtractAttachmentUIDFromName(first.Name)
require.NoError(t, err)
storedFirst, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &firstUID})
require.NoError(t, err)
require.NotNil(t, storedFirst)
require.Equal(t, storepb.AttachmentStorageType_S3, storedFirst.StorageType)
require.Empty(t, storedFirst.Blob)
require.Empty(t, storedFirst.Reference, "S3 attachments must not store a reference URL")
require.Equal(t, oldStorage.Id, storedFirst.Payload.GetS3Object().GetStorageId())
require.Nil(t, storedFirst.Payload.GetS3Object().GetS3Config(), "new attachments must not embed S3 credentials")
oldKey := storedFirst.Payload.GetS3Object().GetKey()
storedInOldBucket, err := objectStore.GetObject("attachments-old", oldKey)
require.NoError(t, err)
require.Equal(t, firstContent, storedInOldBucket)
firstDownloaded, err := ts.Service.GetAttachmentBlob(ctx, storedFirst)
require.NoError(t, err)
require.Equal(t, firstContent, firstDownloaded)
firstMetadata, err := ts.Service.GetAttachment(userCtx, &apipb.GetAttachmentRequest{Name: first.Name})
require.NoError(t, err)
require.Equal(t, first.Name, firstMetadata.Name)
require.Empty(t, firstMetadata.ExternalLink)
newStorage := fakeStorage("s3-new", "New S3", objectStore.Config("attachments-new"))
upsertS3StorageSetting(ctx, t, ts, newStorage.Id, oldStorage, newStorage)
secondContent := []byte("second attachment in new storage")
second, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{
Attachment: &apipb.Attachment{
Filename: "second.txt",
Type: "text/plain",
Content: secondContent,
},
})
require.NoError(t, err)
secondUID, err := api.ExtractAttachmentUIDFromName(second.Name)
require.NoError(t, err)
storedSecond, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &secondUID})
require.NoError(t, err)
require.Equal(t, newStorage.Id, storedSecond.Payload.GetS3Object().GetStorageId())
newKey := storedSecond.Payload.GetS3Object().GetKey()
storedInNewBucket, err := objectStore.GetObject("attachments-new", newKey)
require.NoError(t, err)
require.Equal(t, secondContent, storedInNewBucket)
// Changing the default storage only affects new uploads. The first attachment
// must continue to resolve its original storage by ID.
firstDownloaded, err = ts.Service.GetAttachmentBlob(ctx, storedFirst)
require.NoError(t, err)
require.Equal(t, firstContent, firstDownloaded)
secondDownloaded, err := ts.Service.GetAttachmentBlob(ctx, storedSecond)
require.NoError(t, err)
require.Equal(t, secondContent, secondDownloaded)
_, err = ts.Service.DeleteAttachment(userCtx, &apipb.DeleteAttachmentRequest{Name: first.Name})
require.NoError(t, err)
_, err = objectStore.GetObject("attachments-old", oldKey)
require.Error(t, err, "deleting an attachment must delete its S3 object")
remaining, err := objectStore.GetObject("attachments-new", newKey)
require.NoError(t, err)
require.Equal(t, secondContent, remaining)
}
func TestGetLegacyS3AttachmentBlob(t *testing.T) {
ctx := context.Background()
fake := fakes3.New(t, "legacy-fallback", "legacy-current")
ts := NewTestService(t)
defer ts.Cleanup()
user, err := ts.CreateRegularUser(ctx, "legacy-s3-user")
require.NoError(t, err)
userCtx := ts.CreateUserContext(ctx, user.ID)
t.Run("falls back to embedded config when storage is not registered", func(t *testing.T) {
legacyConfig := fake.Config("legacy-fallback")
driver, err := s3.NewDriver(ctx, legacyConfig)
require.NoError(t, err)
content := []byte("attachment created before storage IDs")
key, err := driver.UploadObject(ctx, "legacy/fallback.txt", "text/plain", bytes.NewReader(content))
require.NoError(t, err)
legacyAttachment, err := ts.Store.CreateAttachment(ctx, &store.Attachment{
UID: "legacy-s3-fallback",
CreatorID: user.ID,
Filename: "fallback.txt",
Type: "text/plain",
Size: int64(len(content)),
StorageType: storepb.AttachmentStorageType_S3,
Payload: &storepb.AttachmentPayload{
Payload: &storepb.AttachmentPayload_S3Object_{
S3Object: &storepb.AttachmentPayload_S3Object{
S3Config: legacyConfig,
Key: key,
},
},
},
})
require.NoError(t, err)
downloaded, err := ts.Service.GetAttachmentBlob(ctx, legacyAttachment)
require.NoError(t, err)
require.Equal(t, content, downloaded)
})
t.Run("uses current storage options for matching namespace", func(t *testing.T) {
currentConfig := fake.Config("legacy-current")
driver, err := s3.NewDriver(ctx, currentConfig)
require.NoError(t, err)
content := []byte("legacy attachment using rotated storage config")
key, err := driver.UploadObject(ctx, "legacy/current.txt", "text/plain", bytes.NewReader(content))
require.NoError(t, err)
configuredStorage := fakeStorage("s3-current", "Current S3", currentConfig)
upsertS3StorageSetting(ctx, t, ts, configuredStorage.Id, configuredStorage)
// Virtual-hosted addressing cannot resolve against httptest's IP endpoint.
// Retrieval only succeeds if the legacy namespace is matched to the named
// storage and its current path-style option is used.
legacyConfig := &storepb.StorageS3Config{
AccessKeyId: "old-access-key",
AccessKeySecret: "old-secret-key",
Endpoint: currentConfig.Endpoint + "/",
Region: currentConfig.Region,
Bucket: currentConfig.Bucket,
UsePathStyle: false,
}
legacyAttachment, err := ts.Store.CreateAttachment(ctx, &store.Attachment{
UID: "legacy-s3-current",
CreatorID: user.ID,
Filename: "current.txt",
Type: "text/plain",
Size: int64(len(content)),
StorageType: storepb.AttachmentStorageType_S3,
Payload: &storepb.AttachmentPayload{
Payload: &storepb.AttachmentPayload_S3Object_{
S3Object: &storepb.AttachmentPayload_S3Object{
S3Config: legacyConfig,
Key: key,
},
},
},
})
require.NoError(t, err)
downloaded, err := ts.Service.GetAttachmentBlob(ctx, legacyAttachment)
require.NoError(t, err)
require.Equal(t, content, downloaded)
_, err = ts.Service.DeleteAttachment(userCtx, &apipb.DeleteAttachmentRequest{Name: api.AttachmentNamePrefix + legacyAttachment.UID})
require.NoError(t, err)
_, err = fake.GetObject(currentConfig.Bucket, key)
require.Error(t, err, "deleting a legacy attachment must use the current matching storage configuration")
})
}
func fakeStorage(id, name string, config *storepb.StorageS3Config) *storepb.Storage {
return &storepb.Storage{
Id: id,
Name: name,
Type: storepb.StorageType_STORAGE_TYPE_S3,
Config: &storepb.Storage_S3Config{S3Config: config},
}
}
func upsertS3StorageSetting(ctx context.Context, t testing.TB, ts *TestService, defaultStorageID string, storages ...*storepb.Storage) {
t.Helper()
_, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
Key: storepb.InstanceSettingKey_STORAGE,
Value: &storepb.InstanceSetting_StorageSetting{
StorageSetting: &storepb.InstanceStorageSetting{
FilepathTemplate: "attachments/{uuid}_{filename}",
Storages: storages,
DefaultStorageId: defaultStorageID,
},
},
})
require.NoError(t, err)
}
// Two uploads with the same filename under a path template that carries no
// random component must not overwrite each other's object.
func TestS3ObjectKeysStayUniqueWithoutUUIDTemplate(t *testing.T) {
ts := NewTestService(t)
defer ts.Cleanup()
fake := fakes3.New(t, "flat")
ctx := context.Background()
storage := fakeStorage("flat-s3", "Flat", fake.Config("flat"))
_, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
Key: storepb.InstanceSettingKey_STORAGE,
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{
FilepathTemplate: "files/{filename}", Storages: []*storepb.Storage{storage}, DefaultStorageId: storage.Id,
}},
})
require.NoError(t, err)
keys := map[string]bool{}
for i, username := range []string{"first-user", "second-user"} {
user, err := ts.CreateRegularUser(ctx, username)
require.NoError(t, err)
userCtx := ts.CreateUserContext(ctx, user.ID)
content := []byte("content of user " + username)
created, err := ts.Service.CreateAttachment(userCtx, &apipb.CreateAttachmentRequest{Attachment: &apipb.Attachment{
Filename: "notes.txt", Type: "text/plain", Content: content,
}})
require.NoError(t, err)
uid, err := api.ExtractAttachmentUIDFromName(created.Name)
require.NoError(t, err)
row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
require.NoError(t, err)
key := row.Payload.GetS3Object().GetKey()
require.False(t, keys[key], "upload %d reused key %q", i, key)
keys[key] = true
stored, err := fake.GetObject("flat", key)
require.NoError(t, err)
require.Equal(t, content, stored)
}
}