Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
fix(server): log file-serving failures and answer missing local files as JSON
boojack3feb08737bfa
diff --git a/internal/httphandler/httphandler.go b/internal/httphandler/httphandler.goindex 62e2526b3..63e52ba08 100644--- a/internal/httphandler/httphandler.go+++ b/internal/httphandler/httphandler.go@@ -34,18 +34,31 @@ func (f Func) ServeHTTP(w http.ResponseWriter, r *http.Request) {}}-// WriteError preserves the public JSON error envelope, including bodyless HEAD errors.+// WriteError answers with the public JSON error envelope, bodyless for HEAD.+// It drops the representation headers a handler may have prepared for the+// response it meant to serve (Content-Type, Content-Disposition and+// Content-Length), so an error is never typed as that body: a media-typed error+// gets handed to the browser's document viewer instead of being read. Other+// headers, such as a 416's Content-Range, survive. The body carries only the+// public message, so a server error's cause is logged here, unless the client+// has already gone away.func WriteError(w http.ResponseWriter, r *http.Request, err error) {code, message := http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)var publicError *Errorif errors.As(err, &publicError) {code, message = publicError.Code, publicError.Message}- // Handlers may have prepared representation headers for the body they- // meant to serve before failing (e.g. the file server sets the attachment's- // Content-Type and Content-Disposition up front). The error body is JSON,- // so the stale values must not leak onto the response: a media-typed error- // gets handed to the browser's document viewer instead of being read.+ if code >= http.StatusInternalServerError && r.Context().Err() == nil {+ cause := err+ if publicError != nil && publicError.Cause != nil {+ cause = publicError.Cause+ }+ slog.Error("HTTP handler error", "method", r.Method, "path", r.URL.Path, "status", code, "message", message, "error", cause)+ }+ writeErrorEnvelope(w, r, code, message)+}++func writeErrorEnvelope(w http.ResponseWriter, r *http.Request, code int, message string) {h := w.Header()h.Del("Content-Type")h.Del("Content-Disposition")@@ -160,7 +173,7 @@ func Recover(next http.Handler) http.Handler {}slog.Error("HTTP handler panic", "panic", recovered, "stack", string(debug.Stack()))if !response.committed {- WriteError(w, r, errors.New("handler panic"))+ writeErrorEnvelope(w, r, http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))}}}()diff --git a/internal/httphandler/httphandler_test.go b/internal/httphandler/httphandler_test.goindex 2b8222c9c..4be4c9fa3 100644--- a/internal/httphandler/httphandler_test.go+++ b/internal/httphandler/httphandler_test.go@@ -120,16 +120,20 @@ func TestJSONFailureBeforeCommit(t *testing.T) {// for the response it meant to serve must not survive onto the error: a// media-typed error gets handed to the browser's document viewer (Chrome's// "Failed to load PDF document" on usememos/memos#6515) instead of read.+// Headers that describe the error itself, such as a 416's Content-Range, stay.func TestErrorUsesJSONContentType(t *testing.T) {rec := httptest.NewRecorder()Func(func(w http.ResponseWriter, _ *http.Request) error {w.Header().Set("Content-Type", "video/mp4")w.Header().Set("Content-Disposition", "attachment")+ w.Header().Set("Content-Length", "10")+ w.Header().Set("Content-Range", "bytes */10")return &Error{Code: http.StatusRequestedRangeNotSatisfiable, Message: "requested range not satisfiable"}}).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))require.Equal(t, http.StatusRequestedRangeNotSatisfiable, rec.Code)require.Equal(t, "application/json", rec.Header().Get("Content-Type"))require.Empty(t, rec.Header().Get("Content-Disposition"))+ require.Equal(t, "bytes */10", rec.Header().Get("Content-Range"))require.JSONEq(t, `{"message":"requested range not satisfiable"}`, rec.Body.String())}diff --git a/server/fileserver/fileserver.go b/server/fileserver/fileserver.goindex fcafcf951..40303a871 100644--- a/server/fileserver/fileserver.go+++ b/server/fileserver/fileserver.go@@ -6,6 +6,7 @@ import ("encoding/base64""fmt""io"+ "io/fs""log/slog""net/http""os"@@ -258,8 +259,7 @@ func (s *FileServerService) serveMediaStream(w http.ResponseWriter, r *http.Requswitch attachment.StorageType {case storepb.AttachmentStorageType_LOCAL:- http.ServeFile(w, r, s.resolveLocalPath(attachment.Reference))- return nil+ return s.serveLocalFile(w, r, attachment)case storepb.AttachmentStorageType_S3:return s.streamS3Object(w, r, attachment, contentType)@@ -297,8 +297,7 @@ func (s *FileServerService) serveStaticFile(w http.ResponseWriter, r *http.Requeswitch attachment.StorageType {case storepb.AttachmentStorageType_LOCAL:- http.ServeFile(w, r, s.resolveLocalPath(attachment.Reference))- return nil+ return s.serveLocalFile(w, r, attachment)case storepb.AttachmentStorageType_S3:return s.streamS3Object(w, r, attachment, contentType)default:@@ -363,6 +362,25 @@ func (s *FileServerService) resolveLocalPath(reference string) string {return filePath}+// serveLocalFile serves an attachment stored on disk. A file that is gone+// answers with the JSON error envelope like every other storage failure,+// instead of the plain-text page net/http would write for it.+func (s *FileServerService) serveLocalFile(w http.ResponseWriter, r *http.Request, attachment *store.Attachment) error {+ path := s.resolveLocalPath(attachment.Reference)+ info, err := os.Stat(path)+ if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ return &httphandler.Error{Code: http.StatusNotFound, Message: "attachment file not found", Cause: err}+ }+ return &httphandler.Error{Code: http.StatusInternalServerError, Message: "failed to read attachment file", Cause: err}+ }+ if info.IsDir() {+ return &httphandler.Error{Code: http.StatusNotFound, Message: "attachment file not found"}+ }+ http.ServeFile(w, r, path)+ return nil+}+// streamS3Object streams S3 content through the server, forwarding a supported// single Range so media players and document viewers can seek without a direct// S3 URL. Multipart ranges are ignored and served as a complete responsediff --git a/server/fileserver/fileserver_test.go b/server/fileserver/fileserver_test.goindex 2bb652a7c..7c8318453 100644--- a/server/fileserver/fileserver_test.go+++ b/server/fileserver/fileserver_test.go@@ -37,12 +37,10 @@ import (teststore "github.com/usememos/memos/store/test")-func TestServeAttachmentFile_S3(t *testing.T) {- ctx := context.Background()- fake := fakes3.New(t, "file-server-attachments")- svc, fs, stores, cleanup := newShareAttachmentTestServices(ctx, t)- defer cleanup()-+// configureFakeS3Storage makes the fake S3 bucket "file-server-attachments"+// the instance's default attachment storage.+func configureFakeS3Storage(ctx context.Context, t *testing.T, stores *store.Store, fake *fakes3.Server) {+ t.Helper()configuredStorage := &storepb.Storage{Id: "s3-files",Name: "File server S3",@@ -58,6 +56,29 @@ func TestServeAttachmentFile_S3(t *testing.T) {}},})require.NoError(t, err)+}++// configureLocalStorage makes the data directory the instance's default+// attachment storage.+func configureLocalStorage(ctx context.Context, t *testing.T, stores *store.Store) {+ t.Helper()+ _, err := stores.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{+ Key: storepb.InstanceSettingKey_STORAGE,+ Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{+ Storages: []*storepb.Storage{{Id: "local", Name: "Local", Type: storepb.StorageType_STORAGE_TYPE_LOCAL}},+ DefaultStorageId: "local",+ }},+ })+ require.NoError(t, err)+}++func TestServeAttachmentFile_S3(t *testing.T) {+ ctx := context.Background()+ fake := fakes3.New(t, "file-server-attachments")+ svc, fs, stores, cleanup := newShareAttachmentTestServices(ctx, t)+ defer cleanup()++ configureFakeS3Storage(ctx, t, stores, fake)creator, err := stores.CreateUser(ctx, &store.User{Username: "s3-file-owner",@@ -111,21 +132,7 @@ func TestServeAttachmentFile_StorageFailureIsAJSONError(t *testing.T) {svc, fs, stores, cleanup := newShareAttachmentTestServices(ctx, t)defer cleanup()- configuredStorage := &storepb.Storage{- Id: "s3-files",- Name: "File server S3",- Type: storepb.StorageType_STORAGE_TYPE_S3,- Config: &storepb.Storage_S3Config{S3Config: fake.Config("file-server-attachments")},- }- _, err := stores.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{- Key: storepb.InstanceSettingKey_STORAGE,- Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{- FilepathTemplate: "files/{uuid}_{filename}",- Storages: []*storepb.Storage{configuredStorage},- DefaultStorageId: configuredStorage.Id,- }},- })- require.NoError(t, err)+ configureFakeS3Storage(ctx, t, stores, fake)creator, err := stores.CreateUser(ctx, &store.User{Username: "s3-failure-owner",@@ -1130,14 +1137,7 @@ func TestThumbnailFailureMarkerOnlyForUnsupportedImages(t *testing.T) {ctx := context.Background()svc, fs, cleanup := newSharedProfileTestServices(ctx, t)defer cleanup()- _, err := svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{- Key: storepb.InstanceSettingKey_STORAGE,- Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{- Storages: []*storepb.Storage{{Id: "local", Name: "Local", Type: storepb.StorageType_STORAGE_TYPE_LOCAL}},- DefaultStorageId: "local",- }},- })- require.NoError(t, err)+ configureLocalStorage(ctx, t, svc.Store)var pngBuf bytes.Bufferrequire.NoError(t, png.Encode(&pngBuf, image.NewRGBA(image.Rect(0, 0, 8, 8))))@@ -1163,3 +1163,32 @@ func TestThumbnailFailureMarkerOnlyForUnsupportedImages(t *testing.T) {require.Equal(t, http.StatusOK, rec.Code)require.Equal(t, "image/jpeg", rec.Header().Get("Content-Type"))}++// A local attachment whose file is gone from disk answers with the JSON error+// envelope like every other storage failure, not net/http's plain-text page.+func TestServeAttachmentFile_MissingLocalFileIsAJSONError(t *testing.T) {+ ctx := context.Background()+ svc, fs, cleanup := newSharedProfileTestServices(ctx, t)+ defer cleanup()+ configureLocalStorage(ctx, t, svc.Store)++ attachment := createPublicImageMemo(ctx, t, svc, "local-owner", "report.pdf", "application/pdf", []byte("%PDF-1.4 local bytes"))+ uid := strings.TrimPrefix(attachment.Name, "attachments/")+ stored, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})+ require.NoError(t, err)+ require.Equal(t, storepb.AttachmentStorageType_LOCAL, stored.StorageType)++ e := http.NewServeMux()+ fs.RegisterRoutes(e, http.HandlerFunc(httphandler.NotFound))+ rec := httptest.NewRecorder()+ e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/report.pdf", attachment.Name), nil))+ require.Equal(t, http.StatusOK, rec.Code)+ require.Equal(t, "application/pdf", rec.Header().Get("Content-Type"))++ require.NoError(t, os.Remove(fs.resolveLocalPath(stored.Reference)))+ rec = httptest.NewRecorder()+ e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/report.pdf", attachment.Name), nil))+ require.Equal(t, http.StatusNotFound, rec.Code)+ require.Equal(t, "application/json", rec.Header().Get("Content-Type"))+ require.JSONEq(t, `{"message":"attachment file not found"}`, rec.Body.String())+}diff --git a/server/fileserver/memo_export.go b/server/fileserver/memo_export.goindex 5784d74a8..99a9f478f 100644--- a/server/fileserver/memo_export.go+++ b/server/fileserver/memo_export.go@@ -78,16 +78,12 @@ func (s *FileServerService) serveMemoExport(w http.ResponseWriter, r *http.Requebody := &commitOnWrite{ResponseWriter: w}if err := s.MemoExporter.WriteMemoExport(ctx, user, body, exportTime); err != nil {- if ctx.Err() == nil {- slog.Error("memo export failed", slog.Int("user", int(user.ID)), slog.Int("caller", int(caller.ID)), slog.Bool("committed", body.committed), slog.Any("error", err))- }- // Before the first byte this becomes a JSON 500, so the archive- // headers must go or the client would save the error as the ZIP.- // After it, the response is already committed and the client- // receives a truncated archive, which the handler wrapper leaves alone.- if !body.committed {- w.Header().Del("Content-Type")- w.Header().Del("Content-Disposition")+ // Before the first byte this becomes a JSON 500, which the handler+ // wrapper logs and strips the archive headers from. After it, the+ // response is already committed and the client receives a truncated+ // archive, which the wrapper leaves alone, so that case is logged here.+ if body.committed && ctx.Err() == nil {+ slog.Error("memo export failed", slog.Int("user", int(user.ID)), slog.Int("caller", int(caller.ID)), slog.Any("error", err))}return &httphandler.Error{Code: http.StatusInternalServerError, Message: "failed to export memos", Cause: err}}