Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

16.0 KB
Raw Blame
package api

import (
	"bytes"
	"context"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/proto"

	"github.com/usememos/memos/internal/testutil"
	apipb "github.com/usememos/memos/proto/gen/api"
	storepb "github.com/usememos/memos/proto/gen/store"
	"github.com/usememos/memos/store"
)

func newUploadTestService(t *testing.T) (*APIService, context.Context) {
	t.Helper()
	svc := newIntegrationService(t)
	t.Cleanup(svc.CloseUploads)
	user := createSpaceTestUser(context.Background(), t, svc, "uploader", store.RoleUser)
	return svc, userCtx(context.Background(), user.ID)
}

func uploadSpec(filename string, size int64) *apipb.UploadAttachmentRequest_Spec {
	return &apipb.UploadAttachmentRequest_Spec{Spec: &apipb.UploadAttachmentSpec{Attachment: &apipb.Attachment{Filename: filename}, TotalSize: size}}
}

func uploadID(id string) *apipb.UploadAttachmentRequest_UploadId {
	return &apipb.UploadAttachmentRequest_UploadId{UploadId: id}
}

func startTestUpload(ctx context.Context, t *testing.T, svc *APIService, size int64) string {
	t.Helper()
	spec := uploadSpec("file.bin", size)
	spec.Spec.Attachment.Type = "application/octet-stream"
	response, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: spec})
	require.NoError(t, err)
	require.NotEmpty(t, response.UploadId)
	require.EqualValues(t, uploadChunkSize, response.MaxChunkSize)
	require.Zero(t, response.CommittedSize)
	require.Nil(t, response.Attachment)
	return response.UploadId
}

func TestUploadAttachmentChunksAndRetries(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	id := startTestUpload(ctx, t, svc, 6)
	first := &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")}
	for range 2 {
		response, err := svc.UploadAttachment(ctx, first)
		require.NoError(t, err)
		require.Equal(t, id, response.UploadId)
		require.EqualValues(t, 3, response.CommittedSize)
	}
	for _, request := range []*apipb.UploadAttachmentRequest{
		{Upload: uploadID(id), Data: []byte("xyz")},
		{Upload: uploadID(id), WriteOffset: 4, Data: []byte("d")},
		{Upload: uploadID(id), WriteOffset: -1, Data: []byte("d")},
	} {
		_, err := svc.UploadAttachment(ctx, request)
		require.Equal(t, codes.OutOfRange, status.Code(err))
	}
	progress, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: -1})
	require.NoError(t, err)
	require.EqualValues(t, 3, progress.CommittedSize)
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, FinishWrite: true})
	require.Equal(t, codes.FailedPrecondition, status.Code(err))

	last := &apipb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, Data: []byte("def"), FinishWrite: true}
	result, err := svc.UploadAttachment(ctx, last)
	require.NoError(t, err)
	require.EqualValues(t, 6, result.Attachment.Size)
	for _, request := range []*apipb.UploadAttachmentRequest{last, {Upload: uploadID(id)}, {Upload: uploadID(id), WriteOffset: 6, FinishWrite: true}} {
		repeated, err := svc.UploadAttachment(ctx, request)
		require.NoError(t, err)
		require.Equal(t, result.Attachment.Name, repeated.Attachment.Name)
	}
	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Len(t, rows, 1)
	content, err := os.ReadFile(rows[0].Reference)
	require.NoError(t, err)
	require.Equal(t, "abcdef", string(content))
	require.NoFileExists(t, svc.attachmentUploads.entries[id].path)

	_, err = svc.DeleteAttachment(ctx, &apipb.DeleteAttachmentRequest{Name: result.Attachment.Name})
	require.NoError(t, err)
	_, err = svc.UploadAttachment(ctx, last)
	require.Equal(t, codes.NotFound, status.Code(err))
}

func TestUploadAttachmentDefaultFileSizeLimit(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	_, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("large.bin", 32*MebiByte+1)})
	require.Equal(t, codes.ResourceExhausted, status.Code(err))
	require.Empty(t, svc.attachmentUploads.entries, "the refused upload creates no session")
	files, err := filepath.Glob(filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"*"))
	require.NoError(t, err)
	require.Empty(t, files, "the refused upload creates no temporary file")
	startTestUpload(ctx, t, svc, 32*MebiByte)
}

func TestUploadAttachmentSingleCall(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	spec := uploadSpec("note.txt", 3)
	spec.Spec.AttachmentId = "chosen-id"
	result, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: spec, Data: []byte("abc"), FinishWrite: true})
	require.NoError(t, err)
	require.Equal(t, "attachments/chosen-id", result.Attachment.Name)
	require.EqualValues(t, 3, result.Attachment.Size)
	require.EqualValues(t, 3, result.CommittedSize)

	empty, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("empty.txt", 0), FinishWrite: true})
	require.NoError(t, err)
	require.Zero(t, empty.Attachment.Size)

	// A spec call may carry the first chunk without finishing.
	partial, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("two.bin", 4), Data: []byte("ab")})
	require.NoError(t, err)
	require.Nil(t, partial.Attachment)
	require.EqualValues(t, 2, partial.CommittedSize)
	done, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(partial.UploadId), WriteOffset: 2, Data: []byte("cd"), FinishWrite: true})
	require.NoError(t, err)
	require.EqualValues(t, 4, done.Attachment.Size)

	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Len(t, rows, 3)
}

func TestUploadAttachmentValidation(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	_, err := svc.UploadAttachment(context.Background(), &apipb.UploadAttachmentRequest{})
	require.Equal(t, codes.Unauthenticated, status.Code(err))
	badID := uploadSpec("file", 0)
	badID.Spec.AttachmentId = "bad id!"
	withContent := uploadSpec("file", 0)
	withContent.Spec.Attachment.Content = []byte("not metadata")
	for _, request := range []*apipb.UploadAttachmentRequest{
		{},
		{Upload: &apipb.UploadAttachmentRequest_Spec{Spec: &apipb.UploadAttachmentSpec{}}},
		{Upload: uploadSpec("file", -1)},
		{Upload: uploadSpec("../file", 0)},
		{Upload: withContent},
		{Upload: badID},
		{Upload: uploadSpec(strings.Repeat("x", attachmentUploadMetadataLimit+1), 0)},
		{Upload: uploadSpec("file", 3), Data: []byte("four")},
	} {
		_, err := svc.UploadAttachment(ctx, request)
		require.Equal(t, codes.InvalidArgument, status.Code(err), request.String())
	}
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("file", 3), WriteOffset: 1, Data: []byte("abc")})
	require.Equal(t, codes.OutOfRange, status.Code(err))
	setDefaultUploadLimitForTest(ctx, t, svc.Store, 1<<30)
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("file", 1<<60)})
	require.Equal(t, codes.ResourceExhausted, status.Code(err))
	require.Empty(t, svc.attachmentUploads.entries, "rejected spec calls must not register uploads")

	id := startTestUpload(ctx, t, svc, 3)
	other := createSpaceTestUser(context.Background(), t, svc, "other-uploader", store.RoleUser)
	_, err = svc.UploadAttachment(userCtx(context.Background(), other.ID), &apipb.UploadAttachmentRequest{Upload: uploadID(id)})
	require.Equal(t, codes.NotFound, status.Code(err))
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID("unknown")})
	require.Equal(t, codes.NotFound, status.Code(err))
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, uploadChunkSize+1)})
	require.Equal(t, codes.ResourceExhausted, status.Code(err))
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("four")})
	require.Equal(t, codes.InvalidArgument, status.Code(err))
	canceled, cancel := context.WithCancel(ctx)
	cancel()
	_, err = svc.UploadAttachment(canceled, &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")})
	require.Error(t, err)
	progress, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id)})
	require.NoError(t, err)
	require.Zero(t, progress.CommittedSize)
}

func TestUploadAttachmentEmptyAndConcurrentFinish(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	id := startTestUpload(ctx, t, svc, 0)
	var wg sync.WaitGroup
	results := make(chan *apipb.UploadAttachmentResponse, 8)
	errs := make(chan error, 8)
	for range 8 {
		wg.Go(func() {
			response, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), FinishWrite: true})
			results <- response
			errs <- err
		})
	}
	wg.Wait()
	close(results)
	close(errs)
	for err := range errs {
		require.NoError(t, err)
	}
	name := ""
	for response := range results {
		if name == "" {
			name = response.Attachment.Name
		}
		require.Equal(t, name, response.Attachment.Name)
		require.Zero(t, response.Attachment.Size)
	}
	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Len(t, rows, 1)
}

func TestUploadAttachmentFinalizationRecovery(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	id := startTestUpload(ctx, t, svc, 3)
	last := &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc"), FinishWrite: true}
	_, err := svc.UploadAttachment(store.WithCreateAttachmentPostCommitFailpoint(ctx), last)
	require.Equal(t, codes.Internal, status.Code(err))
	result, err := svc.UploadAttachment(ctx, last)
	require.NoError(t, err)
	require.NotNil(t, result.Attachment)
	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Len(t, rows, 1)
	content, err := os.ReadFile(rows[0].Reference)
	require.NoError(t, err)
	require.Equal(t, "abc", string(content))
}

func TestUploadAttachmentRechecksMemoAndSize(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	memo, err := svc.CreateMemo(ctx, &apipb.CreateMemoRequest{Memo: &apipb.Memo{Content: "upload target"}})
	require.NoError(t, err)
	spec := uploadSpec("file.txt", 3)
	spec.Spec.Attachment.Memo = &memo.Name
	initial, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: spec})
	require.NoError(t, err)
	_, err = svc.DeleteMemo(ctx, &apipb.DeleteMemoRequest{Name: memo.Name, Force: true})
	require.NoError(t, err)
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: []byte("abc"), FinishWrite: true})
	require.Equal(t, codes.NotFound, status.Code(err))

	// A per-file limit set after the upload started applies at finish.
	id := startTestUpload(ctx, t, svc, 2*MebiByte)
	setDefaultUploadLimitForTest(ctx, t, svc.Store, MebiByte)
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, 2*MebiByte), FinishWrite: true})
	require.Equal(t, codes.ResourceExhausted, status.Code(err))
	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Empty(t, rows)
}

func TestUploadAttachmentExpiryAndLimits(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	ids := make([]string, uploadMaxActivePerUser)
	for i := range ids {
		ids[i] = startTestUpload(ctx, t, svc, 0)
	}
	_, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadSpec("too-many", 0)})
	require.Equal(t, codes.ResourceExhausted, status.Code(err))
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(ids[0]), FinishWrite: true})
	require.NoError(t, err)
	startTestUpload(ctx, t, svc, 0) // Completed uploads do not consume an active slot.
	upload := svc.attachmentUploads.entries[ids[1]]
	upload.mu.Lock()
	upload.expireTime = time.Now().Add(-time.Second)
	upload.mu.Unlock()
	_, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(ids[1])})
	require.Equal(t, codes.NotFound, status.Code(err))

	orphan := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"orphan")
	recent := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"recent")
	unrelated := filepath.Join(svc.Profile.Data, "keep.txt")
	for _, path := range []string{orphan, recent, unrelated} {
		require.NoError(t, os.WriteFile(path, []byte("x"), 0600))
	}
	old := time.Now().Add(-2 * uploadTTL)
	require.NoError(t, os.Chtimes(orphan, old, old))
	require.NoError(t, os.Chtimes(unrelated, old, old))
	svc.attachmentUploads.mu.Lock()
	svc.attachmentUploads.sweepLocked(time.Now(), 0)
	svc.attachmentUploads.removeOrphansLocked(svc.Profile.Data, time.Now())
	svc.attachmentUploads.mu.Unlock()
	require.NoFileExists(t, upload.path)
	require.NoFileExists(t, orphan)
	require.FileExists(t, recent)
	require.FileExists(t, unrelated)
	svc.CloseUploads()
	require.Empty(t, svc.attachmentUploads.entries)
}

func TestUploadAttachmentAboveLegacyRequestLimit(t *testing.T) {
	svc, ctx := newUploadTestService(t)
	setDefaultUploadLimitForTest(ctx, t, svc.Store, 0)
	const size = 300 * MebiByte
	id := startTestUpload(ctx, t, svc, size)
	chunk := bytes.Repeat([]byte{0x5a}, uploadChunkSize)
	var response *apipb.UploadAttachmentResponse
	var err error
	for offset := int64(0); offset < size; offset += int64(len(chunk)) {
		response, err = svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{
			Upload: uploadID(id), WriteOffset: offset, Data: chunk, FinishWrite: offset+int64(len(chunk)) == size,
		})
		require.NoError(t, err)
	}
	require.EqualValues(t, size, response.Attachment.Size)
	rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
	require.NoError(t, err)
	require.Len(t, rows, 1)
	file, err := os.Open(rows[0].Reference)
	require.NoError(t, err)
	defer file.Close()
	info, err := file.Stat()
	require.NoError(t, err)
	require.EqualValues(t, size, info.Size())
	_, err = file.Seek(-int64(len(chunk)), io.SeekEnd)
	require.NoError(t, err)
	tail := make([]byte, len(chunk))
	_, err = io.ReadFull(file, tail)
	require.NoError(t, err)
	require.Equal(t, chunk, tail)
}

func TestUploadAttachmentMediaProcessing(t *testing.T) {
	for _, tc := range []struct {
		name    string
		content []byte
		motion  bool
	}{
		{name: "JPEG", content: testutil.BuildJPEG(20, 10)},
		{name: "motion photo", content: testutil.BuildMotionPhotoJPEG(), motion: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			svc, ctx := newUploadTestService(t)
			metadata := &apipb.Attachment{Filename: "image.unknown", MediaMetadata: &apipb.MediaMetadata{Width: proto.Int32(20), Height: proto.Int32(10)}}
			spec := &apipb.UploadAttachmentRequest_Spec{Spec: &apipb.UploadAttachmentSpec{Attachment: metadata, TotalSize: int64(len(tc.content))}}
			initial, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: spec})
			require.NoError(t, err)
			result, err := svc.UploadAttachment(ctx, &apipb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: tc.content, FinishWrite: true})
			require.NoError(t, err)
			require.Equal(t, "image/jpeg", result.Attachment.Type)
			require.True(t, proto.Equal(metadata.MediaMetadata, result.Attachment.MediaMetadata))
			uid, err := ExtractAttachmentUIDFromName(result.Attachment.Name)
			require.NoError(t, err)
			row, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
			require.NoError(t, err)
			content, err := os.ReadFile(row.Reference)
			require.NoError(t, err)
			if tc.motion {
				require.Equal(t, tc.content, content)
				require.True(t, result.Attachment.MotionMedia.HasEmbeddedVideo)
			} else {
				stripped, err := stripImageExif(bytes.NewReader(tc.content), "image/jpeg")
				require.NoError(t, err)
				require.Equal(t, stripped, content)
			}
		})
	}
}

// setDefaultUploadLimitForTest stores one all-users entry with the given
// per-file limit.
func setDefaultUploadLimitForTest(ctx context.Context, t *testing.T, s *store.Store, uploadBytes int64) {
	t.Helper()
	_, err := s.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
		Key: storepb.InstanceSettingKey_USAGE_LIMITS,
		Value: &storepb.InstanceSetting_UsageLimitsSetting{UsageLimitsSetting: &storepb.InstanceUsageLimitsSetting{
			Limits: []*storepb.InstanceUsageLimitsSetting_UsageLimit{{AllUsers: true, UploadBytes: &uploadBytes}},
		}},
	})
	require.NoError(t, err)
}