Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

fix(frontend): restore sitemap and robots routes

boojackfee7fcd6608b
diff --git a/server/router/frontend/frontend.go b/server/router/frontend/frontend.go
index dedee6390..6034484cd 100644
--- a/server/router/frontend/frontend.go
+++ b/server/router/frontend/frontend.go
@@ -3,10 +3,14 @@ package frontend
import (
"context"
"embed"
+ "encoding/xml"
"io/fs"
+ "net/http"
+ "strings"
"github.com/labstack/echo/v5"
"github.com/labstack/echo/v5/middleware"
+ "github.com/pkg/errors"
"github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/internal/util"
@@ -28,10 +32,10 @@ func NewFrontendService(profile *profile.Profile, store *store.Store) *FrontendS
}
}
-func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {
+func (s *FrontendService) Serve(_ context.Context, e *echo.Echo) {
skipper := func(c *echo.Context) bool {
// Skip API routes.
- if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1") {
+ if util.HasPrefixes(c.Path(), "/api", "/memos.api.v1", "/robots.txt", "/sitemap.xml") {
return true
}
// For index.html and root path, set no-cache headers to prevent browser caching
@@ -57,6 +61,8 @@ func (*FrontendService) Serve(_ context.Context, e *echo.Echo) {
HTML5: true, // Enable fallback to index.html
Skipper: skipper,
}))
+
+ s.registerRoutes(e)
}
func getFileSystem(path string) fs.FS {
@@ -66,3 +72,70 @@ func getFileSystem(path string) fs.FS {
}
return sub
}
+
+func (s *FrontendService) registerRoutes(e *echo.Echo) {
+ e.GET("/robots.txt", s.getRobotsTXT)
+ e.GET("/sitemap.xml", s.getSitemapXML)
+}
+
+func (s *FrontendService) getRobotsTXT(c *echo.Context) error {
+ instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)
+ if err != nil {
+ return err
+ }
+
+ robotsTXT := strings.Join([]string{
+ "User-agent: *",
+ "Allow: /",
+ "Host: " + instanceURL,
+ "Sitemap: " + instanceURL + "/sitemap.xml",
+ }, "\n")
+ return c.String(http.StatusOK, robotsTXT)
+}
+
+func (s *FrontendService) getSitemapXML(c *echo.Context) error {
+ instanceURL, err := normalizeInstanceURL(s.Profile.InstanceURL)
+ if err != nil {
+ return err
+ }
+
+ memos, err := s.Store.ListMemos(c.Request().Context(), &store.FindMemo{
+ VisibilityList: []store.Visibility{store.Public},
+ })
+ if err != nil {
+ return errors.Wrap(err, "failed to list public memos for sitemap")
+ }
+
+ urls := make([]sitemapURL, 0, len(memos))
+ for _, memo := range memos {
+ urls = append(urls, sitemapURL{
+ Loc: instanceURL + "/m/" + memo.UID,
+ })
+ }
+
+ return c.XML(http.StatusOK, sitemapURLSet{
+ XMLNS: sitemapXMLNamespace,
+ URLs: urls,
+ })
+}
+
+func normalizeInstanceURL(instanceURL string) (string, error) {
+ instanceURL = strings.TrimRight(instanceURL, "/")
+ if instanceURL == "" {
+ return "", echo.NewHTTPError(http.StatusNotFound, "instance URL is not configured")
+ }
+ return instanceURL, nil
+}
+
+type sitemapURLSet struct {
+ XMLName xml.Name `xml:"urlset"`
+ XMLNS string `xml:"xmlns,attr"`
+ URLs []sitemapURL `xml:"url"`
+}
+
+type sitemapURL struct {
+ Loc string `xml:"loc"`
+}
+
+//nolint:revive
+const sitemapXMLNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9"
diff --git a/server/router/frontend/frontend_test.go b/server/router/frontend/frontend_test.go
new file mode 100644
index 000000000..912c48136
--- /dev/null
+++ b/server/router/frontend/frontend_test.go
@@ -0,0 +1,93 @@
+package frontend
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/labstack/echo/v5"
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/internal/profile"
+ "github.com/usememos/memos/store"
+ teststore "github.com/usememos/memos/store/test"
+)
+
+func TestFrontendService_RobotsTXT(t *testing.T) {
+ ctx := context.Background()
+ testStore := teststore.NewTestingStore(ctx, t)
+ profile := &profile.Profile{
+ InstanceURL: "https://demo.usememos.com/",
+ }
+
+ e := echo.New()
+ NewFrontendService(profile, testStore).Serve(ctx, e)
+
+ req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil)
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Equal(t, "text/plain; charset=UTF-8", rec.Header().Get("Content-Type"))
+ require.Equal(t, "User-agent: *\nAllow: /\nHost: https://demo.usememos.com\nSitemap: https://demo.usememos.com/sitemap.xml", rec.Body.String())
+}
+
+func TestFrontendService_SitemapXML(t *testing.T) {
+ ctx := context.Background()
+ testStore := teststore.NewTestingStore(ctx, t)
+ profile := &profile.Profile{
+ InstanceURL: "https://demo.usememos.com",
+ }
+
+ user, err := testStore.CreateUser(ctx, &store.User{
+ Username: "sitemap-owner",
+ Role: store.RoleUser,
+ Email: "[email protected]",
+ })
+ require.NoError(t, err)
+
+ _, err = testStore.CreateMemo(ctx, &store.Memo{
+ UID: "publicmemo",
+ CreatorID: user.ID,
+ Content: "public memo",
+ Visibility: store.Public,
+ })
+ require.NoError(t, err)
+
+ _, err = testStore.CreateMemo(ctx, &store.Memo{
+ UID: "privatememo",
+ CreatorID: user.ID,
+ Content: "private memo",
+ Visibility: store.Private,
+ })
+ require.NoError(t, err)
+
+ e := echo.New()
+ NewFrontendService(profile, testStore).Serve(ctx, e)
+
+ req := httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+
+ require.Equal(t, http.StatusOK, rec.Code)
+ require.Contains(t, rec.Header().Get("Content-Type"), "application/xml")
+ require.Contains(t, rec.Body.String(), `<loc>https://demo.usememos.com/m/publicmemo</loc>`)
+ require.NotContains(t, rec.Body.String(), "privatememo")
+}
+
+func TestFrontendService_SitemapRoutesRequireInstanceURL(t *testing.T) {
+ ctx := context.Background()
+ testStore := teststore.NewTestingStore(ctx, t)
+
+ e := echo.New()
+ NewFrontendService(&profile.Profile{}, testStore).Serve(ctx, e)
+
+ for _, path := range []string{"/robots.txt", "/sitemap.xml"} {
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ rec := httptest.NewRecorder()
+ e.ServeHTTP(rec, req)
+
+ require.Equal(t, http.StatusNotFound, rec.Code)
+ }
+}