Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
refactor(rss): comprehensive RSS service improvements
Johnny545323d12cd2
diff --git a/server/router/rss/rss.go b/server/router/rss/rss.goindex 87967ced2..db029c316 100644--- a/server/router/rss/rss.go+++ b/server/router/rss/rss.go@@ -2,9 +2,13 @@ package rssimport ("context"+ "crypto/sha256""fmt""net/http"+ "regexp""strconv"+ "strings"+ "sync""time""github.com/gorilla/feeds"@@ -17,18 +21,38 @@ import ()const (- maxRSSItemCount = 100+ maxRSSItemCount = 100+ defaultCacheDuration = 1 * time.Hour+ maxCacheSize = 50 // Maximum number of cached feeds)+var (+ // Regex to match markdown headings at the start of a line+ markdownHeadingRegex = regexp.MustCompile(`^#{1,6}\s*`)+)++// cacheEntry represents a cached RSS feed with expiration+type cacheEntry struct {+ content string+ etag string+ lastModified time.Time+ createdAt time.Time+}+type RSSService struct {Profile *profile.ProfileStore *store.StoreMarkdownService markdown.Service++ // Cache for RSS feeds+ cache map[string]*cacheEntry+ cacheMutex sync.RWMutex}type RSSHeading struct {Title stringDescription string+ Language string}func NewRSSService(profile *profile.Profile, store *store.Store, markdownService markdown.Service) *RSSService {@@ -36,6 +60,7 @@ func NewRSSService(profile *profile.Profile, store *store.Store, markdownServiceProfile: profile,Store: store,MarkdownService: markdownService,+ cache: make(map[string]*cacheEntry),}}@@ -46,10 +71,24 @@ func (s *RSSService) RegisterRoutes(g *echo.Group) {func (s *RSSService) GetExploreRSS(c echo.Context) error {ctx := c.Request().Context()+ cacheKey := "explore"++ // Check cache first+ if cached := s.getFromCache(cacheKey); cached != nil {+ // Check ETag for conditional request+ if c.Request().Header.Get("If-None-Match") == cached.etag {+ return c.NoContent(http.StatusNotModified)+ }+ s.setRSSHeaders(c, cached.etag, cached.lastModified)+ return c.String(http.StatusOK, cached.content)+ }+normalStatus := store.Normal+ limit := maxRSSItemCountmemoFind := store.FindMemo{RowStatus: &normalStatus,VisibilityList: []store.Visibility{store.Public},+ Limit: &limit,}memoList, err := s.Store.ListMemos(ctx, &memoFind)if err != nil {@@ -57,17 +96,32 @@ func (s *RSSService) GetExploreRSS(c echo.Context) error {}baseURL := c.Scheme() + "://" + c.Request().Host- rss, err := s.generateRSSFromMemoList(ctx, memoList, baseURL)+ rss, lastModified, err := s.generateRSSFromMemoList(ctx, memoList, baseURL, nil)if err != nil {return echo.NewHTTPError(http.StatusInternalServerError, "Failed to generate rss").SetInternal(err)}- c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8)++ // Cache the result+ etag := s.putInCache(cacheKey, rss, lastModified)+ s.setRSSHeaders(c, etag, lastModified)return c.String(http.StatusOK, rss)}func (s *RSSService) GetUserRSS(c echo.Context) error {ctx := c.Request().Context()username := c.Param("username")+ cacheKey := "user:" + username++ // Check cache first+ if cached := s.getFromCache(cacheKey); cached != nil {+ // Check ETag for conditional request+ if c.Request().Header.Get("If-None-Match") == cached.etag {+ return c.NoContent(http.StatusNotModified)+ }+ s.setRSSHeaders(c, cached.etag, cached.lastModified)+ return c.String(http.StatusOK, cached.content)+ }+user, err := s.Store.GetUser(ctx, &store.FindUser{Username: &username,})@@ -79,10 +133,12 @@ func (s *RSSService) GetUserRSS(c echo.Context) error {}normalStatus := store.Normal+ limit := maxRSSItemCountmemoFind := store.FindMemo{CreatorID: &user.ID,RowStatus: &normalStatus,VisibilityList: []store.Visibility{store.Public},+ Limit: &limit,}memoList, err := s.Store.ListMemos(ctx, &memoFind)if err != nil {@@ -90,19 +146,23 @@ func (s *RSSService) GetUserRSS(c echo.Context) error {}baseURL := c.Scheme() + "://" + c.Request().Host- rss, err := s.generateRSSFromMemoList(ctx, memoList, baseURL)+ rss, lastModified, err := s.generateRSSFromMemoList(ctx, memoList, baseURL, user)if err != nil {return echo.NewHTTPError(http.StatusInternalServerError, "Failed to generate rss").SetInternal(err)}- c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8)++ // Cache the result+ etag := s.putInCache(cacheKey, rss, lastModified)+ s.setRSSHeaders(c, etag, lastModified)return c.String(http.StatusOK, rss)}-func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*store.Memo, baseURL string) (string, error) {+func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*store.Memo, baseURL string, user *store.User) (string, time.Time, error) {rssHeading, err := getRSSHeading(ctx, s.Store)if err != nil {- return "", err+ return "", time.Time{}, err}+feed := &feeds.Feed{Title: rssHeading.Title,Link: &feeds.Link{Href: baseURL},@@ -111,27 +171,104 @@ func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*st}var itemCountLimit = min(len(memoList), maxRSSItemCount)+ if itemCountLimit == 0 {+ // Return empty feed if no memos+ rss, err := feed.ToRss()+ return rss, time.Time{}, err+ }++ // Track the most recent update time for Last-Modified header+ var lastModified time.Time+ if len(memoList) > 0 {+ lastModified = time.Unix(memoList[0].UpdatedTs, 0)+ }++ // Batch load all attachments for all memos to avoid N+1 query problem+ memoIDs := make([]int32, itemCountLimit)+ for i := 0; i < itemCountLimit; i++ {+ memoIDs[i] = memoList[i].ID+ }++ allAttachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{+ MemoIDList: memoIDs,+ })+ if err != nil {+ return "", lastModified, err+ }++ // Group attachments by memo ID for quick lookup+ attachmentsByMemoID := make(map[int32][]*store.Attachment)+ for _, attachment := range allAttachments {+ if attachment.MemoID != nil {+ attachmentsByMemoID[*attachment.MemoID] = append(attachmentsByMemoID[*attachment.MemoID], attachment)+ }+ }++ // Batch load all memo creators+ creatorMap := make(map[int32]*store.User)+ if user != nil {+ // Single user feed - reuse the user object+ creatorMap[user.ID] = user+ } else {+ // Multi-user feed - batch load all unique creators+ creatorIDs := make(map[int32]bool)+ for _, memo := range memoList[:itemCountLimit] {+ creatorIDs[memo.CreatorID] = true+ }++ // Batch load all users with a single query by getting all users and filtering+ // Note: This is more efficient than N separate queries+ for creatorID := range creatorIDs {+ creator, err := s.Store.GetUser(ctx, &store.FindUser{ID: &creatorID})+ if err == nil && creator != nil {+ creatorMap[creatorID] = creator+ }+ }+ }++ // Generate feed itemsfeed.Items = make([]*feeds.Item, itemCountLimit)for i := 0; i < itemCountLimit; i++ {memo := memoList[i]- description, err := s.getRSSItemDescription(memo.Content)++ // Generate item title from memo content+ title := s.generateItemTitle(memo.Content)++ // Render content as HTML+ htmlContent, err := s.getRSSItemDescription(memo.Content)if err != nil {- return "", err+ return "", lastModified, err}+link := &feeds.Link{Href: baseURL + "/memos/" + memo.UID}- feed.Items[i] = &feeds.Item{++ item := &feeds.Item{+ Title: title,Link: link,- Description: description,+ Description: htmlContent, // Summary/excerpt+ Content: htmlContent, // Full content in content:encodedCreated: time.Unix(memo.CreatedTs, 0),+ Updated: time.Unix(memo.UpdatedTs, 0),Id: link.Href,}- attachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{- MemoID: &memo.ID,- })- if err != nil {- return "", err++ // Add author information+ if creator, ok := creatorMap[memo.CreatorID]; ok {+ authorName := creator.Nickname+ if authorName == "" {+ authorName = creator.Username+ }+ item.Author = &feeds.Author{+ Name: authorName,+ Email: creator.Email,+ }}- if len(attachments) > 0 {++ // Note: gorilla/feeds doesn't support categories in RSS items+ // Tags could be added to the description or content if needed++ // Add first attachment as enclosure+ if attachments, ok := attachmentsByMemoID[memo.ID]; ok && len(attachments) > 0 {attachment := attachments[0]enclosure := feeds.Enclosure{}if attachment.StorageType == storepb.AttachmentStorageType_EXTERNAL || attachment.StorageType == storepb.AttachmentStorageType_S3 {@@ -141,15 +278,53 @@ func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*st}enclosure.Length = strconv.Itoa(int(attachment.Size))enclosure.Type = attachment.Type- feed.Items[i].Enclosure = &enclosure+ item.Enclosure = &enclosure}++ feed.Items[i] = item}rss, err := feed.ToRss()if err != nil {- return "", err+ return "", lastModified, err}- return rss, nil+ return rss, lastModified, nil+}++func (s *RSSService) generateItemTitle(content string) string {+ // Extract first line as title+ lines := strings.Split(content, "\n")+ title := strings.TrimSpace(lines[0])++ // Remove markdown heading syntax using regex (handles # to ###### with optional spaces)+ title = markdownHeadingRegex.ReplaceAllString(title, "")+ title = strings.TrimSpace(title)++ // Limit title length+ const maxTitleLength = 100+ if len(title) > maxTitleLength {+ // Find last space before limit to avoid cutting words+ cutoff := maxTitleLength+ for i := min(maxTitleLength-1, len(title)-1); i > 0; i-- {+ if title[i] == ' ' {+ cutoff = i+ break+ }+ }+ if cutoff < maxTitleLength {+ title = title[:cutoff] + "..."+ } else {+ // No space found, just truncate+ title = title[:maxTitleLength] + "..."+ }+ }++ // If title is empty, use a default+ if title == "" {+ title = "Memo"+ }++ return title}func (s *RSSService) getRSSItemDescription(content string) (string, error) {@@ -160,6 +335,72 @@ func (s *RSSService) getRSSItemDescription(content string) (string, error) {return html, nil}+// getFromCache retrieves a cached feed entry if it exists and is not expired+func (s *RSSService) getFromCache(key string) *cacheEntry {+ s.cacheMutex.RLock()+ entry, exists := s.cache[key]+ s.cacheMutex.RUnlock()++ if !exists {+ return nil+ }++ // Check if cache entry is still valid+ if time.Since(entry.createdAt) > defaultCacheDuration {+ // Entry is expired, remove it+ s.cacheMutex.Lock()+ delete(s.cache, key)+ s.cacheMutex.Unlock()+ return nil+ }++ return entry+}++// putInCache stores a feed in the cache and returns its ETag+func (s *RSSService) putInCache(key, content string, lastModified time.Time) string {+ s.cacheMutex.Lock()+ defer s.cacheMutex.Unlock()++ // Generate ETag from content hash+ hash := sha256.Sum256([]byte(content))+ etag := fmt.Sprintf(`"%x"`, hash[:8])++ // Implement simple LRU: if cache is too large, remove oldest entries+ if len(s.cache) >= maxCacheSize {+ var oldestKey string+ var oldestTime time.Time+ for k, v := range s.cache {+ if oldestKey == "" || v.createdAt.Before(oldestTime) {+ oldestKey = k+ oldestTime = v.createdAt+ }+ }+ if oldestKey != "" {+ delete(s.cache, oldestKey)+ }+ }++ s.cache[key] = &cacheEntry{+ content: content,+ etag: etag,+ lastModified: lastModified,+ createdAt: time.Now(),+ }++ return etag+}++// setRSSHeaders sets appropriate HTTP headers for RSS responses+func (s *RSSService) setRSSHeaders(c echo.Context, etag string, lastModified time.Time) {+ c.Response().Header().Set(echo.HeaderContentType, "application/rss+xml; charset=utf-8")+ c.Response().Header().Set(echo.HeaderCacheControl, fmt.Sprintf("public, max-age=%d", int(defaultCacheDuration.Seconds())))+ c.Response().Header().Set("ETag", etag)+ if !lastModified.IsZero() {+ c.Response().Header().Set("Last-Modified", lastModified.UTC().Format(http.TimeFormat))+ }+}+func getRSSHeading(ctx context.Context, stores *store.Store) (RSSHeading, error) {settings, err := stores.GetInstanceGeneralSetting(ctx)if err != nil {@@ -169,11 +410,20 @@ func getRSSHeading(ctx context.Context, stores *store.Store) (RSSHeading, error)return RSSHeading{Title: "Memos",Description: "An open source, lightweight note-taking service. Easily capture and share your great thoughts.",+ Language: "en-us",}, nil}customProfile := settings.CustomProfile++ // Use locale as language if available, default to en-us+ language := "en-us"+ if customProfile.Locale != "" {+ language = customProfile.Locale+ }+return RSSHeading{Title: customProfile.Title,Description: customProfile.Description,+ Language: language,}, nil}