Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

feat: pre-signed URL for S3 storage (#2855)

Aleksandr Baryshnikovfa17dce04638
diff --git a/api/v1/resource.go b/api/v1/resource.go
index c4702cae7..b2434f8d4 100644
--- a/api/v1/resource.go
+++ b/api/v1/resource.go
@@ -480,6 +480,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc
Bucket: s3Config.Bucket,
URLPrefix: s3Config.URLPrefix,
URLSuffix: s3Config.URLSuffix,
+ PreSign: s3Config.PreSign,
})
if err != nil {
return errors.Wrap(err, "Failed to create s3 client")
diff --git a/api/v1/storage.go b/api/v1/storage.go
index 4479a1d06..1fc13e043 100644
--- a/api/v1/storage.go
+++ b/api/v1/storage.go
@@ -43,6 +43,7 @@ type StorageS3Config struct {
Bucket string `json:"bucket"`
URLPrefix string `json:"urlPrefix"`
URLSuffix string `json:"urlSuffix"`
+ PreSign bool `json:"presign"`
}
type Storage struct {
diff --git a/api/v1/swagger.yaml b/api/v1/swagger.yaml
index e3576e23c..ad1cf0ec5 100644
--- a/api/v1/swagger.yaml
+++ b/api/v1/swagger.yaml
@@ -238,6 +238,8 @@ definitions:
type: string
urlSuffix:
type: string
+ presign:
+ type: boolean
type: object
api_v1.StorageType:
enum:
@@ -668,6 +670,8 @@ definitions:
type: string
urlSuffix:
type: string
+ presign:
+ type: boolean
type: object
github_com_usememos_memos_api_v1.StorageType:
enum:
diff --git a/bin/memos/main.go b/bin/memos/main.go
index 310ad11f5..98821c85d 100644
--- a/bin/memos/main.go
+++ b/bin/memos/main.go
@@ -12,6 +12,8 @@ import (
"github.com/spf13/viper"
"go.uber.org/zap"
+ "github.com/usememos/memos/internal/jobs"
+
"github.com/usememos/memos/internal/log"
"github.com/usememos/memos/server"
_profile "github.com/usememos/memos/server/profile"
@@ -91,6 +93,9 @@ var (
printGreetings()
+ // update (pre-sign) object storage links if applicable
+ go jobs.RunPreSignLinks(ctx, storeInstance)
+
if err := s.Start(ctx); err != nil {
if err != http.ErrServerClosed {
log.Error("failed to start server", zap.Error(err))
diff --git a/docs/api/v1.md b/docs/api/v1.md
index 48b7704fb..a8426ee7f 100644
--- a/docs/api/v1.md
+++ b/docs/api/v1.md
@@ -1255,6 +1255,7 @@ Get GetImage from URL
| secretKey | string | | No |
| urlPrefix | string | | No |
| urlSuffix | string | | No |
+| presign | boolean | | No |
#### api_v1.StorageType
@@ -1540,16 +1541,18 @@ Get GetImage from URL
#### github_com_usememos_memos_api_v1.StorageS3Config
-| Name | Type | Description | Required |
-| --------- | ------ | ----------- | -------- |
-| accessKey | string | | No |
-| bucket | string | | No |
-| endPoint | string | | No |
-| path | string | | No |
-| region | string | | No |
-| secretKey | string | | No |
-| urlPrefix | string | | No |
-| urlSuffix | string | | No |
+| Name | Type | Description | Required |
+|-----------|---------| ----------- | -------- |
+| accessKey | string | | No |
+| bucket | string | | No |
+| endPoint | string | | No |
+| path | string | | No |
+| region | string | | No |
+| secretKey | string | | No |
+| urlPrefix | string | | No |
+| urlSuffix | string | | No |
+| presign | boolean | | No |
+
#### github_com_usememos_memos_api_v1.StorageType
diff --git a/internal/jobs/presign_link.go b/internal/jobs/presign_link.go
new file mode 100644
index 000000000..36c49bc85
--- /dev/null
+++ b/internal/jobs/presign_link.go
@@ -0,0 +1,140 @@
+package jobs
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "time"
+
+ "github.com/pkg/errors"
+ "go.uber.org/zap"
+
+ apiv1 "github.com/usememos/memos/api/v1"
+ "github.com/usememos/memos/internal/log"
+ "github.com/usememos/memos/plugin/storage/s3"
+ "github.com/usememos/memos/store"
+)
+
+// RunPreSignLinks is a background job that pre-signs external links stored in the database.
+// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.
+func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {
+ for {
+ started := time.Now()
+ if err := signExternalLinks(ctx, dataStore); err != nil {
+ log.Warn("failed sign external links", zap.Error(err))
+ } else {
+ log.Info("links pre-signed", zap.Duration("duration", time.Since(started)))
+ }
+ select {
+ case <-time.After(s3.LinkLifetime / 2):
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func signExternalLinks(ctx context.Context, dataStore *store.Store) error {
+ const pageSize = 32
+
+ objectStore, err := findObjectStorage(ctx, dataStore)
+ if err != nil {
+ return errors.Wrapf(err, "find object storage")
+ }
+ if objectStore == nil || !objectStore.Config.PreSign {
+ // object storage not set or not supported
+ return nil
+ }
+
+ var offset int
+ var limit = pageSize
+ for {
+ resources, err := dataStore.ListResources(ctx, &store.FindResource{
+ GetBlob: false,
+ Limit: &limit,
+ Offset: &offset,
+ })
+ if err != nil {
+ return errors.Wrapf(err, "list resources, offset %d", offset)
+ }
+
+ for _, res := range resources {
+ if res.ExternalLink == "" {
+ // not for object store
+ continue
+ }
+ if strings.Contains(res.ExternalLink, "?") && time.Since(time.Unix(res.UpdatedTs, 0)) < s3.LinkLifetime/2 {
+ // resource not signed (hack for migration)
+ // resource was recently updated - skipping
+ continue
+ }
+ newLink, err := objectStore.PreSignLink(ctx, res.ExternalLink)
+ if err != nil {
+ log.Warn("failed pre-sign link", zap.Int32("resource", res.ID), zap.String("link", res.ExternalLink), zap.Error(err))
+ continue // do not fail - we may want update left over links too
+ }
+ now := time.Now().Unix()
+ // we may want to use here transaction and batch update in the future
+ _, err = dataStore.UpdateResource(ctx, &store.UpdateResource{
+ ID: res.ID,
+ UpdatedTs: &now,
+ ExternalLink: &newLink,
+ })
+ if err != nil {
+ // something with DB - better to stop here
+ return errors.Wrapf(err, "update resource %d link to %q", res.ID, newLink)
+ }
+ }
+
+ offset += limit
+ if len(resources) < limit {
+ break
+ }
+ }
+ return nil
+}
+
+// findObjectStorage returns current default storage if it's S3-compatible or nil otherwise.
+// Returns error only in case of internal problems (ie: database or configuration issues).
+// May return nil client and nil error.
+func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client, error) {
+ systemSettingStorageServiceID, err := dataStore.GetSystemSetting(ctx, &store.FindSystemSetting{Name: apiv1.SystemSettingStorageServiceIDName.String()})
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to find SystemSettingStorageServiceIDName")
+ }
+
+ storageServiceID := apiv1.DefaultStorage
+ if systemSettingStorageServiceID != nil {
+ err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to unmarshal storage service id")
+ }
+ }
+ storage, err := dataStore.GetStorage(ctx, &store.FindStorage{ID: &storageServiceID})
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to find StorageServiceID")
+ }
+
+ if storage == nil {
+ return nil, nil // storage not configured - not an error, just return empty ref
+ }
+ storageMessage, err := apiv1.ConvertStorageFromStore(storage)
+
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to ConvertStorageFromStore")
+ }
+ if storageMessage.Type != apiv1.StorageS3 {
+ return nil, nil
+ }
+
+ s3Config := storageMessage.Config.S3Config
+ return s3.NewClient(ctx, &s3.Config{
+ AccessKey: s3Config.AccessKey,
+ SecretKey: s3Config.SecretKey,
+ EndPoint: s3Config.EndPoint,
+ Region: s3Config.Region,
+ Bucket: s3Config.Bucket,
+ URLPrefix: s3Config.URLPrefix,
+ URLSuffix: s3Config.URLSuffix,
+ PreSign: s3Config.PreSign,
+ })
+}
diff --git a/plugin/storage/s3/s3.go b/plugin/storage/s3/s3.go
index 8618fd623..842032c40 100644
--- a/plugin/storage/s3/s3.go
+++ b/plugin/storage/s3/s3.go
@@ -7,6 +7,7 @@ import (
"io"
"net/url"
"strings"
+ "time"
"github.com/aws/aws-sdk-go-v2/aws"
s3config "github.com/aws/aws-sdk-go-v2/config"
@@ -14,8 +15,11 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
+ errors2 "github.com/pkg/errors"
)
+const LinkLifetime = 24 * time.Hour
+
type Config struct {
AccessKey string
SecretKey string
@@ -24,6 +28,7 @@ type Config struct {
Region string
URLPrefix string
URLSuffix string
+ PreSign bool
}
type Client struct {
@@ -93,5 +98,44 @@ func (client *Client) UploadFile(ctx context.Context, filename string, fileType
if link == "" {
return "", errors.New("failed to get file link")
}
+ if client.Config.PreSign {
+ return client.PreSignLink(ctx, link)
+ }
return link, nil
}
+
+// PreSignLink generates a pre-signed URL for the given sourceLink.
+// If the link does not belong to the configured storage endpoint, it is returned as-is.
+// If the link belongs to the storage, the function generates a pre-signed URL using the AWS S3 client.
+func (client *Client) PreSignLink(ctx context.Context, sourceLink string) (string, error) {
+ u, err := url.Parse(sourceLink)
+ if err != nil {
+ return "", errors2.Wrapf(err, "parse URL")
+ }
+ // if link doesn't belong to storage, then return as-is.
+ // the empty hostname is corner-case for AWS native endpoint.
+ if client.Config.EndPoint != "" && !strings.Contains(client.Config.EndPoint, u.Hostname()) {
+ return sourceLink, nil
+ }
+
+ filename := u.Path
+ if prefixLen := len(client.Config.URLPrefix); len(filename) >= prefixLen {
+ filename = filename[prefixLen:]
+ }
+ if suffixLen := len(client.Config.URLSuffix); len(filename) >= suffixLen {
+ filename = filename[:len(filename)-suffixLen]
+ }
+ filename = strings.Trim(filename, "/")
+ if strings.HasPrefix(filename, client.Config.Bucket) {
+ filename = strings.Trim(filename[len(client.Config.Bucket):], "/")
+ }
+
+ req, err := awss3.NewPresignClient(client.Client).PresignGetObject(ctx, &awss3.GetObjectInput{
+ Bucket: aws.String(client.Config.Bucket),
+ Key: aws.String(filename),
+ }, awss3.WithPresignExpires(LinkLifetime))
+ if err != nil {
+ return "", errors2.Wrapf(err, "pre-sign link")
+ }
+ return req.URL, nil
+}
diff --git a/store/db/mysql/resource.go b/store/db/mysql/resource.go
index 76321e484..e2c570eae 100644
--- a/store/db/mysql/resource.go
+++ b/store/db/mysql/resource.go
@@ -133,6 +133,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
if v := update.InternalPath; v != nil {
set, args = append(set, "`internal_path` = ?"), append(args, *v)
}
+ if v := update.ExternalLink; v != nil {
+ set, args = append(set, "`external_link` = ?"), append(args, *v)
+ }
if v := update.MemoID; v != nil {
set, args = append(set, "`memo_id` = ?"), append(args, *v)
}
diff --git a/store/db/postgres/resource.go b/store/db/postgres/resource.go
index 9409b14d6..3ced74615 100644
--- a/store/db/postgres/resource.go
+++ b/store/db/postgres/resource.go
@@ -118,6 +118,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
if v := update.InternalPath; v != nil {
set, args = append(set, "internal_path = "+placeholder(len(args)+1)), append(args, *v)
}
+ if v := update.ExternalLink; v != nil {
+ set, args = append(set, "external_link = "+placeholder(len(args)+1)), append(args, *v)
+ }
if v := update.MemoID; v != nil {
set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v)
}
diff --git a/store/db/sqlite/resource.go b/store/db/sqlite/resource.go
index 136098980..bb0d6ee78 100644
--- a/store/db/sqlite/resource.go
+++ b/store/db/sqlite/resource.go
@@ -114,6 +114,9 @@ func (d *DB) UpdateResource(ctx context.Context, update *store.UpdateResource) (
if v := update.InternalPath; v != nil {
set, args = append(set, "`internal_path` = ?"), append(args, *v)
}
+ if v := update.ExternalLink; v != nil {
+ set, args = append(set, "`external_link` = ?"), append(args, *v)
+ }
if v := update.MemoID; v != nil {
set, args = append(set, "`memo_id` = ?"), append(args, *v)
}
diff --git a/store/resource.go b/store/resource.go
index 371700845..fae1cd1b5 100644
--- a/store/resource.go
+++ b/store/resource.go
@@ -53,6 +53,7 @@ type UpdateResource struct {
UpdatedTs *int64
Filename *string
InternalPath *string
+ ExternalLink *string
MemoID *int32
Blob []byte
}
diff --git a/web/src/components/CreateStorageServiceDialog.tsx b/web/src/components/CreateStorageServiceDialog.tsx
index c3fe68290..d9be9dac8 100644
--- a/web/src/components/CreateStorageServiceDialog.tsx
+++ b/web/src/components/CreateStorageServiceDialog.tsx
@@ -1,5 +1,5 @@
-import { Button, IconButton, Input, Typography } from "@mui/joy";
-import { useEffect, useState } from "react";
+import { Button, IconButton, Input, Checkbox, Typography } from "@mui/joy";
+import React, { useEffect, useState } from "react";
import { toast } from "react-hot-toast";
import * as api from "@/helpers/api";
import { useTranslate } from "@/utils/i18n";
@@ -29,6 +29,7 @@ const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
bucket: "",
urlPrefix: "",
urlSuffix: "",
+ presign: false,
});
const isCreating = storage === undefined;
@@ -220,6 +221,12 @@ const CreateStorageServiceDialog: React.FC<Props> = (props: Props) => {
onChange={(e) => setPartialS3Config({ urlSuffix: e.target.value })}
fullWidth
/>
+ <Checkbox
+ className="mb-2"
+ label={t("setting.storage-section.presign-placeholder")}
+ checked={s3Config.presign}
+ onChange={(e) => setPartialS3Config({ presign: e.target.checked })}
+ />
<div className="mt-2 w-full flex flex-row justify-end items-center space-x-1">
<Button variant="plain" color="neutral" onClick={handleCloseBtnClick}>
{t("common.cancel")}
diff --git a/web/src/locales/en.json b/web/src/locales/en.json
index f35d05459..41773d29d 100644
--- a/web/src/locales/en.json
+++ b/web/src/locales/en.json
@@ -228,7 +228,8 @@
"url-prefix": "URL prefix",
"url-prefix-placeholder": "Custom URL prefix, optional",
"url-suffix": "URL suffix",
- "url-suffix-placeholder": "Custom URL suffix, optional"
+ "url-suffix-placeholder": "Custom URL suffix, optional",
+ "presign-placeholder": "Pre-sign URL, optional"
},
"member-section": {
"create-a-member": "Create a member",
diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json
index 58781b614..ffb6c3425 100644
--- a/web/src/locales/ru.json
+++ b/web/src/locales/ru.json
@@ -302,6 +302,7 @@
"update-storage": "Обновить хранилище",
"url-prefix": "Префикс URL",
"url-prefix-placeholder": "Пользовательский префикс URL, необязательно",
+ "presign-placeholder": "Генерировать временную публичную ссылку, необязательно",
"url-suffix": "суффикс URL",
"url-suffix-placeholder": "Пользовательский суффикс URL, необязательно",
"warning-text": "Вы уверены, что хотите удалить это хранилище?\nЭТО ДЕЙСТВИЕ НЕВОЗМОЖНО ОТМЕНИТЬ❗"
diff --git a/web/src/types/modules/storage.d.ts b/web/src/types/modules/storage.d.ts
index fe0f8e9a4..fc9a8c540 100644
--- a/web/src/types/modules/storage.d.ts
+++ b/web/src/types/modules/storage.d.ts
@@ -11,6 +11,7 @@ interface StorageS3Config {
bucket: string;
urlPrefix: string;
urlSuffix: string;
+ presign: boolean;
}
interface StorageConfig {