Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve restic: prevent concurrent append-only overwrites

Vasek Sraierd632f8bb5188
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 4a2ed7e16..e72801b3b 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -13,6 +13,7 @@ import (
"path"
"regexp"
"strings"
+ "sync"
"time"
"github.com/go-chi/chi/v5"
@@ -283,19 +284,28 @@ func checkPrivate(next http.Handler) http.Handler {
})
}
+// uploadLock serializes uploads to one remote.
+type uploadLock struct {
+ uploadingMu sync.Mutex
+ refs int // protected by server.uploadLocksMu
+}
+
// server contains everything to run the server
type server struct {
- server *libhttp.Server
- f fs.Fs
- cache *cache
- opt Options
+ server *libhttp.Server
+ f fs.Fs
+ cache *cache
+ opt Options
+ uploadLocksMu sync.Mutex
+ uploadLocks map[string]*uploadLock
}
func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *server, err error) {
s = &server{
- f: f,
- cache: newCache(opt.CacheObjects),
- opt: *opt,
+ f: f,
+ cache: newCache(opt.CacheObjects),
+ opt: *opt,
+ uploadLocks: make(map[string]*uploadLock),
}
// Don't bind any HTTP listeners if running with --stdio
if opt.Stdio {
@@ -313,6 +323,30 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *server, err error
return s, nil
}
+// lockUpload locks uploads to remote and returns its unlock function.
+func (s *server) lockUpload(remote string) func() {
+ s.uploadLocksMu.Lock()
+ lock := s.uploadLocks[remote]
+ if lock == nil {
+ lock = new(uploadLock)
+ s.uploadLocks[remote] = lock
+ }
+ lock.refs++
+ s.uploadLocksMu.Unlock()
+
+ lock.uploadingMu.Lock()
+ return func() {
+ lock.uploadingMu.Unlock()
+
+ s.uploadLocksMu.Lock()
+ lock.refs--
+ if lock.refs == 0 {
+ delete(s.uploadLocks, remote)
+ }
+ s.uploadLocksMu.Unlock()
+ }
+}
+
// Serve restic until the server is shutdown
func (s *server) Serve() error {
s.server.Serve()
@@ -422,12 +456,19 @@ func (s *server) postObject(w http.ResponseWriter, r *http.Request) {
return
}
if s.opt.AppendOnly {
- // make sure the file does not exist yet
+ unlock := s.lockUpload(remote)
+ defer unlock()
+
+ // Only a definitive not-found result permits creating the object.
_, err := s.newObject(r.Context(), remote)
if err == nil {
fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode")
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
-
+ return
+ }
+ if !errors.Is(err, fs.ErrorObjectNotFound) {
+ fs.Errorf(remote, "Post request: failed to check whether file exists: %v", err)
+ http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
}
diff --git a/cmd/serve/restic/restic_appendonly_test.go b/cmd/serve/restic/restic_appendonly_test.go
index af5e8e680..e79b5c6ea 100644
--- a/cmd/serve/restic/restic_appendonly_test.go
+++ b/cmd/serve/restic/restic_appendonly_test.go
@@ -4,13 +4,17 @@ import (
"context"
"crypto/rand"
"encoding/hex"
+ "errors"
"io"
"net/http"
+ "net/http/httptest"
"strings"
"testing"
"github.com/rclone/rclone/cmd"
+ "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configfile"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -135,3 +139,60 @@ func TestResticHandler(t *testing.T) {
})
}
}
+
+func TestAppendOnlyFailsClosedOnLookupError(t *testing.T) {
+ ctx := context.Background()
+ configfile.Install()
+
+ underlying := cmd.NewFsSrc([]string{t.TempDir()})
+ f := &newObjectErrorFs{
+ Fs: underlying,
+ err: errors.New("lookup failed"),
+ }
+ opt := newOpt()
+ opt.AppendOnly = true
+ s, err := newServer(ctx, f, &opt)
+ require.NoError(t, err)
+
+ req := newRequest(t, "POST", "/config", strings.NewReader("replacement"))
+ checkRequest(t, s.server.Router().ServeHTTP, req, []wantFunc{wantCode(http.StatusInternalServerError)})
+
+ _, err = underlying.NewObject(ctx, "config")
+ assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}
+
+func TestAppendOnlyConcurrentUploads(t *testing.T) {
+ configfile.Install()
+ opt := newOpt()
+ opt.AppendOnly = true
+ f := cmd.NewFsSrc([]string{t.TempDir()})
+ s, err := newServer(context.Background(), f, &opt)
+ require.NoError(t, err)
+ handler := s.server.Router().ServeHTTP
+
+ const uploads = 10
+ start := make(chan struct{})
+ results := make(chan int, uploads)
+ for range uploads {
+ go func() {
+ req := newRequest(t, "POST", "/config", strings.NewReader("config data"))
+ recorder := httptest.NewRecorder()
+ <-start
+ handler(recorder, req)
+ results <- recorder.Code
+ }()
+ }
+ close(start)
+
+ succeeded := 0
+ for range uploads {
+ if code := <-results; code == http.StatusOK {
+ succeeded++
+ } else {
+ assert.Equal(t, http.StatusForbidden, code)
+ }
+ }
+ assert.Equal(t, 1, succeeded)
+ checkRequest(t, handler, newRequest(t, "GET", "/config", nil),
+ []wantFunc{wantCode(http.StatusOK), wantBody("config data")})
+}