Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
fix(frontend): restore sitemap and robots routes
boojackfee7fcd6608b
diff --git a/server/router/frontend/frontend.go b/server/router/frontend/frontend.goindex dedee6390..6034484cd 100644--- a/server/router/frontend/frontend.go+++ b/server/router/frontend/frontend.go@@ -3,10 +3,14 @@ package frontendimport ("context""embed"+ "encoding/xml""io/fs"+ "net/http"+ "strings""github.com/labstack/echo/v5""github.com/labstack/echo/v5/middleware"+ "github.com/pkg/errors""github.com/usememos/memos/internal/profile""github.com/usememos/memos/internal/util"@@ -28,10 +32,10 @@ func NewFrontendService(profile *profile.Profile, store *store.Store) *FrontendS}}-func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {+func (s *FrontendService) Serve(_ context.Context, e *echo.Echo) {skipper := func(c *echo.Context) bool {// Skip API routes.- if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1") {+ if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1", "/robots.txt", "/sitemap.xml") {return true}// For index.html and root path, set no-cache headers to prevent browser caching@@ -57,6 +61,8 @@ func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {HTML5: true, // Enable fallback to index.htmlSkipper: skipper,}))++ s.registerRoutes(e)}func getFileSystem(path string) fs.FS {@@ -66,3 +72,70 @@ func getFileSystem(path string) fs.FS {}return sub}++func (s *FrontendService) registerRoutes(e *echo.Echo) {+ e.GET("/robots.txt", s.getRobotsTXT)+ e.GET("/sitemap.xml", s.getSitemapXML)+}++func (s *FrontendService) getRobotsTXT(c *echo.Context) error {+ instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)+ if err != nil {+ return err+ }++ robotsTXT := strings.Join([]string{+ "User-agent: *",+ "Allow: /",+ "Host: " + instanceURL,+ "Sitemap: " + instanceURL + "/sitemap.xml",+ }, "\n")+ return c.String(http.StatusOK, robotsTXT)+}++func (s *FrontendService) getSitemapXML(c *echo.Context) error {+ instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)+ if err != nil {+ return err+ }++ memos, err := s.Store.ListMemos(c.Request().Context(), &store.FindMemo{+ VisibilityList: []store.Visibility{store.Public},+ })+ if err != nil {+ return errors.Wrap(err, "failed to list public memos for sitemap")+ }++ urls := make([]sitemapURL, 0, len(memos))+ for _, memo := range memos {+ urls = append(urls, sitemapURL{+ Loc: instanceURL + "/m/" + memo.UID,+ })+ }++ return c.XML(http.StatusOK, sitemapURLSet{+ XMLNS: sitemapXMLNamespace,+ URLs: urls,+ })+}++func normalizeInstanceURL(instanceURL string) (string, error) {+ instanceURL = strings.TrimRight(instanceURL, "/")+ if instanceURL == "" {+ return "", echo.NewHTTPError(http.StatusNotFound, "instance URL is not configured")+ }+ return instanceURL, nil+}++type sitemapURLSet struct {+ XMLName xml.Name `xml:"urlset"`+ XMLNS string `xml:"xmlns,attr"`+ URLs []sitemapURL `xml:"url"`+}++type sitemapURL struct {+ Loc string `xml:"loc"`+}++//nolint:revive+const sitemapXMLNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9"diff --git a/server/router/frontend/frontend_test.go b/server/router/frontend/frontend_test.gonew file mode 100644index 000000000..912c48136--- /dev/null+++ b/server/router/frontend/frontend_test.go@@ -0,0 +1,93 @@+package frontend++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/labstack/echo/v5"+ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/internal/profile"+ "github.com/usememos/memos/store"+ teststore "github.com/usememos/memos/store/test"+)++func TestFrontendService_RobotsTXT(t *testing.T) {+ ctx := context.Background()+ testStore := teststore.NewTestingStore(ctx, t)+ profile := &profile.Profile{+ InstanceURL: "https://demo.usememos.com/",+ }++ e := echo.New()+ NewFrontendService(profile, testStore).Serve(ctx, e)++ req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)+ rec := httptest.NewRecorder()+ e.ServeHTTP(rec, req)++ require.Equal(t, http.StatusOK, rec.Code)+ require.Equal(t, "text/plain; charset=UTF-8", rec.Header().Get("Content-Type"))+ require.Equal(t, "User-agent: *\nAllow: /\nHost: https://demo.usememos.com\nSitemap: https://demo.usememos.com/sitemap.xml", rec.Body.String())+}++func TestFrontendService_SitemapXML(t *testing.T) {+ ctx := context.Background()+ testStore := teststore.NewTestingStore(ctx, t)+ profile := &profile.Profile{+ InstanceURL: "https://demo.usememos.com",+ }++ user, err := testStore.CreateUser(ctx, &store.User{+ Username: "sitemap-owner",+ Role: store.RoleUser,+ Email: "[email protected]",+ })+ require.NoError(t, err)++ _, err = testStore.CreateMemo(ctx, &store.Memo{+ UID: "publicmemo",+ CreatorID: user.ID,+ Content: "public memo",+ Visibility: store.Public,+ })+ require.NoError(t, err)++ _, err = testStore.CreateMemo(ctx, &store.Memo{+ UID: "privatememo",+ CreatorID: user.ID,+ Content: "private memo",+ Visibility: store.Private,+ })+ require.NoError(t, err)++ e := echo.New()+ NewFrontendService(profile, testStore).Serve(ctx, e)++ req := httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)+ rec := httptest.NewRecorder()+ e.ServeHTTP(rec, req)++ require.Equal(t, http.StatusOK, rec.Code)+ require.Contains(t, rec.Header().Get("Content-Type"), "application/xml")+ require.Contains(t, rec.Body.String(), `<loc>https://demo.usememos.com/m/publicmemo</loc>`)+ require.NotContains(t, rec.Body.String(), "privatememo")+}++func TestFrontendService_SitemapRoutesRequireInstanceURL(t *testing.T) {+ ctx := context.Background()+ testStore := teststore.NewTestingStore(ctx, t)++ e := echo.New()+ NewFrontendService(&profile.Profile{}, testStore).Serve(ctx, e)++ for _, path := range []string{"/robots.txt", "/sitemap.xml"} {+ req := httptest.NewRequest(http.MethodGet, path, nil)+ rec := httptest.NewRecorder()+ e.ServeHTTP(rec, req)++ require.Equal(t, http.StatusNotFound, rec.Code)+ }+}