Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
chore: update background services
Stevene8dfd579c363
diff --git a/bin/memos/main.go b/bin/memos/main.goindex f4ad21e4c..0feec6b04 100644--- a/bin/memos/main.go+++ b/bin/memos/main.go@@ -84,7 +84,7 @@ var (printGreetings()- if err := s.Start(); err != nil {+ if err := s.Start(ctx); err != nil {if err != http.ErrServerClosed {slog.Error("failed to start server", err)cancel()diff --git a/server/server.go b/server/server.goindex 83ea92f8e..eb4b668cf 100644--- a/server/server.go+++ b/server/server.go@@ -15,7 +15,6 @@ import ("github.com/soheilhy/cmux""google.golang.org/grpc"- "github.com/usememos/memos/internal/jobs"storepb "github.com/usememos/memos/proto/gen/store""github.com/usememos/memos/server/profile""github.com/usememos/memos/server/route/api/auth"@@ -23,6 +22,7 @@ import ("github.com/usememos/memos/server/route/frontend""github.com/usememos/memos/server/route/resource""github.com/usememos/memos/server/route/rss"+ resourcepresign "github.com/usememos/memos/server/service/resource_presign"versionchecker "github.com/usememos/memos/server/service/version_checker""github.com/usememos/memos/store")@@ -104,7 +104,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Storereturn s, nil}-func (s *Server) Start() error {+func (s *Server) Start(ctx context.Context) error {address := fmt.Sprintf(":%d", s.Profile.Port)listener, err := net.Listen("tcp", address)if err != nil {@@ -125,6 +125,7 @@ func (s *Server) Start() error {slog.Error("failed to start echo server", err)}}()+ s.StartBackgroundRunners(ctx)return muxServer.Serve()}@@ -146,8 +147,8 @@ func (s *Server) Shutdown(ctx context.Context) {fmt.Printf("memos stopped properly\n")}-func (s *Server) StartRunners(ctx context.Context) {- go jobs.RunPreSignLinks(ctx, s.Store)+func (s *Server) StartBackgroundRunners(ctx context.Context) {+ go resourcepresign.RunPreSignLinks(ctx, s.Store)go versionchecker.NewVersionChecker(s.Store, s.Profile).Start(ctx)}diff --git a/internal/jobs/presign_link.go b/server/service/resource_presign/resource_presign.gosimilarity index 56%rename from internal/jobs/presign_link.gorename to server/service/resource_presign/resource_presign.goindex 23868044b..9186d2efc 100644--- a/internal/jobs/presign_link.go+++ b/server/service/resource_presign/resource_presign.go@@ -1,4 +1,4 @@-package jobs+package resourcepresignimport ("context"@@ -9,13 +9,11 @@ import ("github.com/pkg/errors""github.com/usememos/memos/plugin/storage/s3"- apiv2pb "github.com/usememos/memos/proto/gen/api/v2"storepb "github.com/usememos/memos/proto/gen/store"- apiv2 "github.com/usememos/memos/server/route/api/v2""github.com/usememos/memos/store")-// RunPreSignLinks is a background job that pre-signs external links stored in the database.+// RunPreSignLinks is a background runner that pre-signs external links stored in the database.// It uses S3 client to generate presigned URLs and updates the corresponding resources in the store.func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {for {@@ -33,8 +31,6 @@ func RunPreSignLinks(ctx context.Context, dataStore *store.Store) {}func signExternalLinks(ctx context.Context, dataStore *store.Store) error {- const pageSize = 32-objectStore, err := findObjectStorage(ctx, dataStore)if err != nil {return errors.Wrapf(err, "find object storage")@@ -44,51 +40,40 @@ func signExternalLinks(ctx context.Context, dataStore *store.Store) error {return nil}- var offset int- var limit = pageSize- for {- resources, err := dataStore.ListResources(ctx, &store.FindResource{- GetBlob: false,- Limit: &limit,- Offset: &offset,- })- if err != nil {- return errors.Wrapf(err, "list resources, offset %d", offset)+ resources, err := dataStore.ListResources(ctx, &store.FindResource{+ GetBlob: false,+ })+ if err != nil {+ return errors.Wrapf(err, "list resources")+ }++ for _, resource := range resources {+ if resource.ExternalLink == "" {+ // not for object store+ continue+ }+ if strings.Contains(resource.ExternalLink, "?") && time.Since(time.Unix(resource.UpdatedTs, 0)) < s3.LinkLifetime/2 {+ // resource not signed (hack for migration)+ // resource was recently updated - skipping+ continue}- for _, res := range resources {- if res.ExternalLink == "" {- // not for object store- continue- }- if strings.Contains(res.ExternalLink, "?") && time.Since(time.Unix(res.UpdatedTs, 0)) < s3.LinkLifetime/2 {- // resource not signed (hack for migration)- // resource was recently updated - skipping- continue- }- newLink, err := objectStore.PreSignLink(ctx, res.ExternalLink)- if err != nil {- slog.Error("failed to pre-sign link", err)- continue // do not fail - we may want update left over links too- }- now := time.Now().Unix()- // we may want to use here transaction and batch update in the future- _, err = dataStore.UpdateResource(ctx, &store.UpdateResource{- ID: res.ID,- UpdatedTs: &now,- ExternalLink: &newLink,- })- if err != nil {- // something with DB - better to stop here- return errors.Wrapf(err, "update resource %d link to %q", res.ID, newLink)- }+ newLink, err := objectStore.PreSignLink(ctx, resource.ExternalLink)+ if err != nil {+ slog.Error("failed to pre-sign link", err)+ continue}- offset += limit- if len(resources) < limit {- break+ now := time.Now().Unix()+ if _, err := dataStore.UpdateResource(ctx, &store.UpdateResource{+ ID: resource.ID,+ UpdatedTs: &now,+ ExternalLink: &newLink,+ }); err != nil {+ return errors.Wrapf(err, "update resource %d link to %q", resource.ID, newLink)}}+return nil}@@ -107,16 +92,11 @@ func findObjectStorage(ctx context.Context, dataStore *store.Store) (*s3.Client,if err != nil {return nil, errors.Wrap(err, "Failed to find storage")}- if storage == nil {- return nil, nil- }-- storageMessage := apiv2.ConvertStorageFromStore(storage)- if storageMessage.Type != apiv2pb.Storage_S3 {+ if storage == nil || storage.Type != storepb.Storage_S3 {return nil, nil}- s3Config := storageMessage.Config.GetS3Config()+ s3Config := storage.Config.GetS3Config()return s3.NewClient(ctx, &s3.Config{AccessKey: s3Config.AccessKey,SecretKey: s3Config.SecretKey,