Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

chore: split save resource asset (#1939)

Athurg Gooth06dbd8731161
diff --git a/api/v1/resource.go b/api/v1/resource.go
index d9109d982..0fdb34c0a 100644
--- a/api/v1/resource.go
+++ b/api/v1/resource.go
@@ -119,7 +119,6 @@ func (s *APIV1Service) registerResourceRoutes(g *echo.Group) {
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to read %s", request.ExternalLink))
}
- create.Blob = blob
mediaType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err != nil {
@@ -136,6 +135,12 @@ func (s *APIV1Service) registerResourceRoutes(g *echo.Group) {
}
create.Filename = filename
create.ExternalLink = ""
+ create.Size = int64(len(blob))
+
+ err = SaveResourceBlob(ctx, s.Store, create, bytes.NewReader(blob))
+ if err != nil {
+ return echo.NewHTTPError(http.StatusInternalServerError, "Failed to save resource").SetInternal(err)
+ }
}
}
@@ -182,129 +187,21 @@ func (s *APIV1Service) registerResourceRoutes(g *echo.Group) {
return echo.NewHTTPError(http.StatusBadRequest, "Failed to parse upload data").SetInternal(err)
}
- filetype := file.Header.Get("Content-Type")
- size := file.Size
sourceFile, err := file.Open()
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "Failed to open file").SetInternal(err)
}
defer sourceFile.Close()
- systemSettingStorageServiceID, err := s.Store.GetSystemSetting(ctx, &store.FindSystemSetting{Name: SystemSettingStorageServiceIDName.String()})
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err)
- }
- storageServiceID := DatabaseStorage
- if systemSettingStorageServiceID != nil {
- err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal storage service id").SetInternal(err)
- }
+ create := &store.Resource{
+ CreatorID: userID,
+ Filename: file.Filename,
+ Type: file.Header.Get("Content-Type"),
+ Size: file.Size,
}
-
- var create *store.Resource
- if storageServiceID == DatabaseStorage {
- fileBytes, err := io.ReadAll(sourceFile)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to read file").SetInternal(err)
- }
- create = &store.Resource{
- CreatorID: userID,
- Filename: file.Filename,
- Type: filetype,
- Size: size,
- Blob: fileBytes,
- }
- } else if storageServiceID == LocalStorage {
- // filepath.Join() should be used for local file paths,
- // as it handles the os-specific path separator automatically.
- // path.Join() always uses '/' as path separator.
- systemSettingLocalStoragePath, err := s.Store.GetSystemSetting(ctx, &store.FindSystemSetting{Name: SystemSettingLocalStoragePathName.String()})
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find local storage path setting").SetInternal(err)
- }
- localStoragePath := "assets/{filename}"
- if systemSettingLocalStoragePath != nil && systemSettingLocalStoragePath.Value != "" {
- err = json.Unmarshal([]byte(systemSettingLocalStoragePath.Value), &localStoragePath)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to unmarshal local storage path setting").SetInternal(err)
- }
- }
- filePath := filepath.FromSlash(localStoragePath)
- if !strings.Contains(filePath, "{filename}") {
- filePath = filepath.Join(filePath, "{filename}")
- }
- filePath = filepath.Join(s.Profile.Data, replacePathTemplate(filePath, file.Filename))
-
- dir := filepath.Dir(filePath)
- if err = os.MkdirAll(dir, os.ModePerm); err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create directory").SetInternal(err)
- }
- dst, err := os.Create(filePath)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to create file").SetInternal(err)
- }
- defer dst.Close()
- _, err = io.Copy(dst, sourceFile)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to copy file").SetInternal(err)
- }
-
- create = &store.Resource{
- CreatorID: userID,
- Filename: file.Filename,
- Type: filetype,
- Size: size,
- InternalPath: filePath,
- }
- } else {
- storage, err := s.Store.GetStorage(ctx, &store.FindStorage{ID: &storageServiceID})
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find storage").SetInternal(err)
- }
- if storage == nil {
- return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Storage %d not found", storageServiceID))
- }
- storageMessage, err := ConvertStorageFromStore(storage)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to convert storage").SetInternal(err)
- }
-
- if storageMessage.Type == StorageS3 {
- s3Config := storageMessage.Config.S3Config
- s3Client, err := s3.NewClient(ctx, &s3.Config{
- AccessKey: s3Config.AccessKey,
- SecretKey: s3Config.SecretKey,
- EndPoint: s3Config.EndPoint,
- Region: s3Config.Region,
- Bucket: s3Config.Bucket,
- URLPrefix: s3Config.URLPrefix,
- URLSuffix: s3Config.URLSuffix,
- })
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to new s3 client").SetInternal(err)
- }
-
- filePath := s3Config.Path
- if !strings.Contains(filePath, "{filename}") {
- filePath = path.Join(filePath, "{filename}")
- }
- filePath = replacePathTemplate(filePath, file.Filename)
- _, filename := filepath.Split(filePath)
- link, err := s3Client.UploadFile(ctx, filePath, filetype, sourceFile)
- if err != nil {
- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to upload via s3 client").SetInternal(err)
- }
- create = &store.Resource{
- CreatorID: userID,
- Filename: filename,
- Type: filetype,
- Size: size,
- ExternalLink: link,
- }
- } else {
- return echo.NewHTTPError(http.StatusInternalServerError, "Unsupported storage type")
- }
+ err = SaveResourceBlob(ctx, s.Store, create, sourceFile)
+ if err != nil {
+ return echo.NewHTTPError(http.StatusInternalServerError, "Failed to save resource").SetInternal(err)
}
resource, err := s.Store.CreateResource(ctx, create)
@@ -420,7 +317,7 @@ func (s *APIV1Service) registerResourceRoutes(g *echo.Group) {
}
ext := filepath.Ext(resource.Filename)
- thumbnailPath := path.Join(s.Profile.Data, thumbnailImagePath, fmt.Sprintf("%d%s", resource.ID, ext))
+ thumbnailPath := filepath.Join(s.Profile.Data, thumbnailImagePath, fmt.Sprintf("%d%s", resource.ID, ext))
if err := os.Remove(thumbnailPath); err != nil {
log.Warn(fmt.Sprintf("failed to delete local thumbnail with path %s", thumbnailPath), zap.Error(err))
}
@@ -485,7 +382,7 @@ func (s *APIV1Service) registerResourcePublicRoutes(g *echo.Group) {
if c.QueryParam("thumbnail") == "1" && util.HasPrefixes(resource.Type, "image/png", "image/jpeg") {
ext := filepath.Ext(resource.Filename)
- thumbnailPath := path.Join(s.Profile.Data, thumbnailImagePath, fmt.Sprintf("%d%s", resource.ID, ext))
+ thumbnailPath := filepath.Join(s.Profile.Data, thumbnailImagePath, fmt.Sprintf("%d%s", resource.ID, ext))
thumbnailBlob, err := getOrGenerateThumbnailImage(blob, thumbnailPath)
if err != nil {
log.Warn(fmt.Sprintf("failed to get or generate local thumbnail with path %s", thumbnailPath), zap.Error(err))
@@ -659,3 +556,116 @@ func convertResourceFromStore(resource *store.Resource) *Resource {
LinkedMemoAmount: resource.LinkedMemoAmount,
}
}
+
+// SaveResourceBlob save the blob of resource based on the storage config
+//
+// Depend on the storage config, some fields of *store.ResourceCreate will be changed:
+// 1. *DatabaseStorage*: `create.Blob`.
+// 2. *LocalStorage*: `create.InternalPath`.
+// 3. Others( external service): `create.ExternalLink`.
+func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resource, r io.Reader) error {
+ systemSettingStorageServiceID, err := s.GetSystemSetting(ctx, &store.FindSystemSetting{Name: SystemSettingStorageServiceIDName.String()})
+ if err != nil {
+ return fmt.Errorf("Failed to find SystemSettingStorageServiceIDName: %s", err)
+ }
+
+ storageServiceID := DatabaseStorage
+ if systemSettingStorageServiceID != nil {
+ err = json.Unmarshal([]byte(systemSettingStorageServiceID.Value), &storageServiceID)
+ if err != nil {
+ return fmt.Errorf("Failed to unmarshal storage service id: %s", err)
+ }
+ }
+
+ // `DatabaseStorage` means store blob into database
+ if storageServiceID == DatabaseStorage {
+ fileBytes, err := io.ReadAll(r)
+ if err != nil {
+ return fmt.Errorf("Failed to read file: %s", err)
+ }
+ create.Blob = fileBytes
+ return nil
+ }
+
+ // `LocalStorage` means save blob into local disk
+ if storageServiceID == LocalStorage {
+ systemSettingLocalStoragePath, err := s.GetSystemSetting(ctx, &store.FindSystemSetting{Name: SystemSettingLocalStoragePathName.String()})
+ if err != nil {
+ return fmt.Errorf("Failed to find SystemSettingLocalStoragePathName: %s", err)
+ }
+ localStoragePath := "assets/{filename}"
+ if systemSettingLocalStoragePath != nil && systemSettingLocalStoragePath.Value != "" {
+ err = json.Unmarshal([]byte(systemSettingLocalStoragePath.Value), &localStoragePath)
+ if err != nil {
+ return fmt.Errorf("Failed to unmarshal SystemSettingLocalStoragePathName: %s", err)
+ }
+ }
+ filePath := filepath.FromSlash(localStoragePath)
+ if !strings.Contains(filePath, "{filename}") {
+ filePath = filepath.Join(filePath, "{filename}")
+ }
+ filePath = filepath.Join(s.Profile.Data, replacePathTemplate(filePath, create.Filename))
+
+ dir := filepath.Dir(filePath)
+ if err = os.MkdirAll(dir, os.ModePerm); err != nil {
+ return fmt.Errorf("Failed to create directory: %s", err)
+ }
+ dst, err := os.Create(filePath)
+ if err != nil {
+ return fmt.Errorf("Failed to create file: %s", err)
+ }
+ defer dst.Close()
+ _, err = io.Copy(dst, r)
+ if err != nil {
+ return fmt.Errorf("Failed to copy file: %s", err)
+ }
+
+ create.InternalPath = filePath
+ return nil
+ }
+
+ // Others: store blob into external service, such as S3
+ storage, err := s.GetStorage(ctx, &store.FindStorage{ID: &storageServiceID})
+ if err != nil {
+ return fmt.Errorf("Failed to find StorageServiceID: %s", err)
+ }
+ if storage == nil {
+ return fmt.Errorf("Storage %d not found", storageServiceID)
+ }
+ storageMessage, err := ConvertStorageFromStore(storage)
+ if err != nil {
+ return fmt.Errorf("Failed to ConvertStorageFromStore: %s", err)
+ }
+
+ if storageMessage.Type != StorageS3 {
+ return fmt.Errorf("Unsupported storage type: %s", storageMessage.Type)
+ }
+
+ s3Config := storageMessage.Config.S3Config
+ s3Client, err := s3.NewClient(ctx, &s3.Config{
+ AccessKey: s3Config.AccessKey,
+ SecretKey: s3Config.SecretKey,
+ EndPoint: s3Config.EndPoint,
+ Region: s3Config.Region,
+ Bucket: s3Config.Bucket,
+ URLPrefix: s3Config.URLPrefix,
+ URLSuffix: s3Config.URLSuffix,
+ })
+ if err != nil {
+ return fmt.Errorf("Failed to create s3 client: %s", err)
+ }
+
+ filePath := s3Config.Path
+ if !strings.Contains(filePath, "{filename}") {
+ filePath = filepath.Join(filePath, "{filename}")
+ }
+ filePath = replacePathTemplate(filePath, create.Filename)
+
+ link, err := s3Client.UploadFile(ctx, filePath, create.Type, r)
+ if err != nil {
+ return fmt.Errorf("Failed to upload via s3 client: %s", err)
+ }
+
+ create.ExternalLink = link
+ return nil
+}
diff --git a/server/telegram.go b/server/telegram.go
index b0b4dd837..b3b1da1b0 100644
--- a/server/telegram.go
+++ b/server/telegram.go
@@ -1,6 +1,7 @@
package server
import (
+ "bytes"
"context"
"encoding/json"
"fmt"
@@ -84,13 +85,21 @@ func (t *telegramHandler) MessageHandle(ctx context.Context, bot *telegram.Bot,
// create resources
for _, attachment := range attachments {
- resource, err := t.store.CreateResource(ctx, &store.Resource{
+ // Fill the common field of create
+ create := store.Resource{
CreatorID: creatorID,
Filename: attachment.FileName,
Type: attachment.GetMimeType(),
Size: attachment.FileSize,
- Blob: attachment.Data,
- })
+ }
+
+ err := apiv1.SaveResourceBlob(ctx, t.store, &create, bytes.NewReader(attachment.Data))
+ if err != nil {
+ _, err := bot.EditMessage(ctx, message.Chat.ID, reply.MessageID, fmt.Sprintf("failed to SaveResourceBlob: %s", err), nil)
+ return err
+ }
+
+ resource, err := t.store.CreateResource(ctx, &create)
if err != nil {
_, err := bot.EditMessage(ctx, message.Chat.ID, reply.MessageID, fmt.Sprintf("failed to CreateResource: %s", err), nil)
return err