Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

12.3 KB
Raw Blame
package api

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"mime"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"time"

	"github.com/pkg/errors"
	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/usememos/memos/internal/profile"
	"github.com/usememos/memos/internal/random"
	apipb "github.com/usememos/memos/proto/gen/api"
	storepb "github.com/usememos/memos/proto/gen/store"
	"github.com/usememos/memos/store"
)

func convertAttachmentFromStore(attachment *store.Attachment) *apipb.Attachment {
	attachmentMessage := &apipb.Attachment{
		Name:          fmt.Sprintf("%s%s", AttachmentNamePrefix, attachment.UID),
		CreateTime:    timestamppb.New(time.Unix(attachment.CreatedTs, 0)),
		Filename:      attachment.Filename,
		Type:          attachment.Type,
		Size:          attachment.Size,
		MotionMedia:   convertMotionMediaFromStore(getAttachmentMotionMedia(attachment)),
		MediaMetadata: convertMediaMetadataFromStore(attachment.Payload.GetMediaMetadata()),
	}
	if attachment.CreatorUsername != nil {
		attachmentMessage.Creator = BuildUserName(*attachment.CreatorUsername)
	}
	if attachment.MemoUID != nil && *attachment.MemoUID != "" {
		memoName := fmt.Sprintf("%s%s", MemoNamePrefix, *attachment.MemoUID)
		attachmentMessage.Memo = &memoName
	}
	// Managed storage is always addressed through the authenticated file route.
	// In particular, never expose an expiring S3 presigned URL as attachment API
	// metadata because it can outlive a memo visibility change.
	if attachment.StorageType == storepb.AttachmentStorageType_EXTERNAL {
		attachmentMessage.ExternalLink = attachment.Reference
	}

	return attachmentMessage
}

// attachmentContextReader stops a long local copy once the request context is
// canceled; io.Copy has no context of its own, unlike the S3 client.
type attachmentContextReader struct {
	ctx    context.Context
	reader io.Reader
}

func (r *attachmentContextReader) Read(p []byte) (int, error) {
	if err := r.ctx.Err(); err != nil {
		return 0, err
	}
	return r.reader.Read(p)
}

// saveAttachmentContent writes content to the default storage and records
// where it went on create. Database storage keeps the bytes on create.Blob.
func saveAttachmentContent(
	ctx context.Context,
	profile *profile.Profile,
	stores *store.Store,
	create *store.Attachment,
	instanceStorageSetting *storepb.InstanceStorageSetting,
	content io.Reader,
) error {
	defaultStorage := store.GetDefaultStorage(instanceStorageSetting)
	if defaultStorage == nil {
		return errors.New("default storage is not configured")
	}

	if defaultStorage.Type == storepb.StorageType_STORAGE_TYPE_LOCAL {
		filepathTemplate := "assets/{timestamp}_{uuid}_{filename}"
		if instanceStorageSetting.FilepathTemplate != "" {
			filepathTemplate = instanceStorageSetting.FilepathTemplate
		}

		internalPath := filepathTemplate
		if !strings.Contains(internalPath, "{filename}") {
			internalPath = filepath.Join(internalPath, "{filename}")
		}
		internalPath = replaceFilenameWithPathTemplate(internalPath, create.Filename)
		internalPath = filepath.ToSlash(internalPath)

		// Ensure the directory exists.
		osPath := filepath.FromSlash(internalPath)
		if !filepath.IsAbs(osPath) {
			osPath = filepath.Join(profile.Data, osPath)
		}
		osPath = ensureUniqueLocalAttachmentPath(osPath, create.UID)
		internalPath = filepath.ToSlash(osPath)
		if !filepath.IsAbs(filepath.FromSlash(internalPath)) {
			relativePath, err := filepath.Rel(profile.Data, osPath)
			if err != nil {
				return errors.Wrap(err, "Failed to get relative path")
			}
			internalPath = filepath.ToSlash(relativePath)
		}
		dir := filepath.Dir(osPath)
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			return errors.Wrap(err, "Failed to create directory")
		}

		// Stage in a temp file so partial content never appears at the final path.
		file, err := os.CreateTemp(dir, ".memos-upload-*")
		if err != nil {
			return errors.Wrap(err, "failed to create attachment file")
		}
		defer os.Remove(file.Name())
		defer file.Close()
		if _, err := io.Copy(file, &attachmentContextReader{ctx: ctx, reader: content}); err != nil {
			return errors.Wrap(err, "failed to write attachment file")
		}
		if err := file.Chmod(0644); err != nil {
			return errors.Wrap(err, "failed to set attachment permissions")
		}
		if err := file.Close(); err != nil {
			return errors.Wrap(err, "failed to close attachment file")
		}
		if err := os.Rename(file.Name(), osPath); err != nil {
			return errors.Wrap(err, "failed to finalize attachment file")
		}
		create.Reference = internalPath
		create.Blob = nil
		create.StorageType = storepb.AttachmentStorageType_LOCAL
	} else if defaultStorage.Type == storepb.StorageType_STORAGE_TYPE_S3 {
		driver, err := stores.StorageDriver(ctx, defaultStorage)
		if err != nil {
			return errors.Wrap(err, "failed to create storage driver")
		}

		filepathTemplate := instanceStorageSetting.FilepathTemplate
		if !strings.Contains(filepathTemplate, "{filename}") {
			filepathTemplate = filepath.Join(filepathTemplate, "{filename}")
		}
		objectKey := replaceFilenameWithPathTemplate(filepathTemplate, create.Filename)
		if !strings.Contains(filepathTemplate, "{uuid}") {
			// PutObject overwrites silently, and two users can pick the same
			// filename in the same second, so a template without a random
			// component gets the attachment UID appended to keep keys unique.
			ext := filepath.Ext(objectKey)
			objectKey = strings.TrimSuffix(objectKey, ext) + "_" + create.UID + ext
		}
		key, err := driver.UploadObject(ctx, objectKey, create.Type, content)
		if err != nil {
			return errors.Wrap(err, "failed to upload via storage driver")
		}

		// S3 attachments carry no reference; they are served via the authenticated file route.
		create.Blob = nil
		create.StorageType = storepb.AttachmentStorageType_S3
		payload := ensureAttachmentPayload(create.Payload)
		payload.Payload = &storepb.AttachmentPayload_S3Object_{
			S3Object: &storepb.AttachmentPayload_S3Object{
				Key:       key,
				StorageId: defaultStorage.Id,
			},
		}
		create.Payload = payload
	} else {
		blob, err := io.ReadAll(content)
		if err != nil {
			return errors.Wrap(err, "failed to read attachment content")
		}
		create.Blob = blob
	}

	return nil
}

func cleanupSavedAttachmentBlob(
	ctx context.Context,
	stores *store.Store,
	attachment *store.Attachment,
	instanceStorageSetting *storepb.InstanceStorageSetting,
) (bool, error) {
	if attachment == nil || (attachment.StorageType != storepb.AttachmentStorageType_LOCAL && attachment.StorageType != storepb.AttachmentStorageType_S3) {
		return false, nil
	}

	cleanupCtx, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
	defer cancelCleanup()
	persisted, err := stores.GetAttachment(cleanupCtx, &store.FindAttachment{UID: &attachment.UID})
	if err != nil {
		return false, errors.Wrap(err, "failed to verify attachment create outcome")
	}
	if hasSameManagedStorageObject(persisted, attachment) {
		return true, nil
	}
	return false, stores.DeleteAttachmentStorageWithInstanceSetting(cleanupCtx, attachment, instanceStorageSetting)
}

func hasSameManagedStorageObject(left, right *store.Attachment) bool {
	if left == nil || right == nil || left.StorageType != right.StorageType {
		return false
	}
	switch left.StorageType {
	case storepb.AttachmentStorageType_LOCAL:
		return left.Reference != "" && left.Reference == right.Reference
	case storepb.AttachmentStorageType_S3:
		leftObject, rightObject := left.Payload.GetS3Object(), right.Payload.GetS3Object()

		if leftObject == nil || rightObject == nil {
			return false
		}

		if leftObject.Key == "" || rightObject.Key == "" {
			return false
		}

		return leftObject.Key == rightObject.Key && leftObject.StorageId == rightObject.StorageId
	default:
		return false
	}
}

func (s *APIService) cleanupDeletedAttachmentStorage(ctx context.Context, attachments []*store.Attachment) error {
	var instanceStorageSetting *storepb.InstanceStorageSetting
	var instanceStorageSettingErr error
	if slices.ContainsFunc(attachments, store.AttachmentNeedsInstanceStorageSetting) {
		instanceStorageSetting, instanceStorageSettingErr = s.Store.GetInstanceStorageSetting(ctx)
	}

	var firstErr error
	for _, attachment := range attachments {
		if attachment == nil {
			continue
		}
		var err error
		if instanceStorageSettingErr != nil && store.AttachmentNeedsInstanceStorageSetting(attachment) {
			err = errors.Wrap(instanceStorageSettingErr, "failed to get instance storage setting")
		} else {
			err = s.Store.DeleteAttachmentStorageWithInstanceSetting(ctx, attachment, instanceStorageSetting)
		}
		if err != nil && firstErr == nil {
			firstErr = errors.Wrapf(err, "attachment %d", attachment.ID)
		}
	}
	return firstErr
}

// GetAttachmentBlob reads an attachment from its configured storage.
func (s *APIService) GetAttachmentBlob(ctx context.Context, attachment *store.Attachment) ([]byte, error) {
	content, err := s.openAttachmentContent(ctx, attachment)
	if err != nil {
		return nil, err
	}
	defer content.Close()
	blob, err := io.ReadAll(content)
	if err != nil {
		return nil, errors.Wrap(err, "failed to read attachment content")
	}
	return blob, nil
}

// openAttachmentContent streams an attachment from its configured storage.
// Database-backed attachments are re-read with their blob when the caller
// listed them without it.
func (s *APIService) openAttachmentContent(ctx context.Context, attachment *store.Attachment) (io.ReadCloser, error) {
	switch attachment.StorageType {
	case storepb.AttachmentStorageType_LOCAL:
		attachmentPath := filepath.FromSlash(attachment.Reference)
		if !filepath.IsAbs(attachmentPath) {
			attachmentPath = filepath.Join(s.Profile.Data, attachmentPath)
		}
		file, err := os.Open(attachmentPath)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, errors.Wrap(err, "file not found")
			}
			return nil, errors.Wrap(err, "failed to open the file")
		}
		return file, nil
	case storepb.AttachmentStorageType_S3:
		driver, s3Object, err := s.Store.ResolveAttachmentS3Driver(ctx, attachment)
		if err != nil {
			return nil, errors.Wrap(err, "failed to resolve S3 attachment driver")
		}
		object, err := driver.GetObjectStream(ctx, s3Object.Key, "")
		if err != nil {
			return nil, errors.Wrap(err, "failed to stream object from S3")
		}
		return object.Body, nil
	default:
		blob := attachment.Blob
		if blob == nil {
			stored, err := s.Store.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID, GetBlob: true})
			if err != nil {
				return nil, errors.Wrap(err, "failed to load attachment blob")
			}
			if stored == nil {
				return nil, errors.New("attachment not found")
			}
			blob = stored.Blob
		}
		return io.NopCloser(bytes.NewReader(blob)), nil
	}
}

var fileKeyPattern = regexp.MustCompile(`\{[a-z]{1,9}\}`)

func replaceFilenameWithPathTemplate(path, filename string) string {
	t := time.Now()
	path = fileKeyPattern.ReplaceAllStringFunc(path, func(s string) string {
		switch s {
		case "{filename}":
			return filename
		case "{timestamp}":
			return fmt.Sprintf("%d", t.Unix())
		case "{year}":
			return fmt.Sprintf("%d", t.Year())
		case "{month}":
			return fmt.Sprintf("%02d", t.Month())
		case "{day}":
			return fmt.Sprintf("%02d", t.Day())
		case "{hour}":
			return fmt.Sprintf("%02d", t.Hour())
		case "{minute}":
			return fmt.Sprintf("%02d", t.Minute())
		case "{second}":
			return fmt.Sprintf("%02d", t.Second())
		case "{uuid}":
			return random.UUID()
		default:
			return s
		}
	})
	return path
}

func ensureUniqueLocalAttachmentPath(path, uid string) string {
	if _, err := os.Stat(path); err != nil {
		return path
	}

	ext := filepath.Ext(path)
	base := strings.TrimSuffix(path, ext)
	return base + "_" + uid + ext
}

func validateFilename(filename string) bool {
	// Reject path traversal attempts and make sure no additional directories are created
	if !filepath.IsLocal(filename) || strings.ContainsAny(filename, "/\\") {
		return false
	}

	// Reject filenames starting or ending with spaces or periods
	if strings.HasPrefix(filename, " ") || strings.HasSuffix(filename, " ") ||
		strings.HasPrefix(filename, ".") || strings.HasSuffix(filename, ".") {
		return false
	}

	return true
}

func normalizeMimeType(mimeType string) (string, bool) {
	mimeType = strings.TrimSpace(mimeType)
	if mimeType == "" || len(mimeType) > 255 {
		return "", false
	}

	mediaType, _, err := mime.ParseMediaType(mimeType)
	if err != nil || mediaType == "" || len(mediaType) > 255 {
		return "", false
	}

	return mediaType, true
}