Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve sftp: fix file corruption when a client resumes an upload

Nick Craig-Wood4dead760dd4a
diff --git a/cmd/serve/sftp/handler.go b/cmd/serve/sftp/handler.go
index f7c9c3c47..985b5998a 100644
--- a/cmd/serve/sftp/handler.go
+++ b/cmd/serve/sftp/handler.go
@@ -38,7 +38,22 @@ func (v vfsHandler) Fileread(r *sftp.Request) (io.ReaderAt, error) {
}
func (v vfsHandler) Filewrite(r *sftp.Request) (io.WriterAt, error) {
- file, err := v.OpenFile(r.Filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+ // Respect the flags requested in the SFTP OPEN packet
+ p := r.Pflags()
+ flags := os.O_WRONLY
+ if p.Append {
+ flags |= os.O_APPEND
+ }
+ if p.Creat {
+ flags |= os.O_CREATE
+ }
+ if p.Trunc {
+ flags |= os.O_TRUNC
+ }
+ if p.Excl {
+ flags |= os.O_EXCL
+ }
+ file, err := v.OpenFile(r.Filepath, flags, 0777)
if err != nil {
return nil, err
}
diff --git a/cmd/serve/sftp/handler_test.go b/cmd/serve/sftp/handler_test.go
new file mode 100644
index 000000000..1b91c8dc0
--- /dev/null
+++ b/cmd/serve/sftp/handler_test.go
@@ -0,0 +1,153 @@
+// Test the SFTP serve handler against a real server and client.
+//
+// We skip tests on platforms with troublesome character mappings
+
+//go:build !windows && !darwin && !plan9
+
+package sftp
+
+import (
+ "context"
+ "io"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/pkg/sftp"
+ _ "github.com/rclone/rclone/backend/local"
+ "github.com/rclone/rclone/cmd/serve/proxy"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/vfs/vfscommon"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/crypto/ssh"
+)
+
+// startTestServer starts an sftp server serving a temporary local directory
+// with the given VFS options and returns a connected sftp client.
+func startTestServer(t *testing.T, vfsOpt *vfscommon.Options) *sftp.Client {
+ ctx := context.Background()
+
+ f, err := fs.NewFs(ctx, t.TempDir())
+ require.NoError(t, err)
+
+ opt := Opt
+ opt.ListenAddr = testBindAddress
+ opt.User = testUser
+ opt.Pass = testPass
+
+ w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)
+ require.NoError(t, err)
+ go func() {
+ _ = w.Serve()
+ }()
+ t.Cleanup(func() {
+ assert.NoError(t, w.Shutdown())
+ })
+
+ clientConfig := &ssh.ClientConfig{
+ User: testUser,
+ Auth: []ssh.AuthMethod{ssh.Password(testPass)},
+ HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+ }
+ conn, err := ssh.Dial("tcp", w.Addr().String(), clientConfig)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = conn.Close()
+ })
+
+ client, err := sftp.NewClient(conn)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = client.Close()
+ })
+
+ return client
+}
+
+// Test that re-opening a file for write without the truncate flag preserves
+// the data already written, rather than zeroing the start of the file.
+//
+// This reproduces a corruption seen with WinSCP "Process in Background", which
+// resumes an upload on a second connection by re-opening the partial file
+// without SSH_FXF_TRUNC and writing from the offset it had reached.
+func TestFilewriteResumeNoTruncate(t *testing.T) {
+ vfsOpt := vfscommon.Opt
+ vfsOpt.CacheMode = vfscommon.CacheModeWrites
+ client := startTestServer(t, &vfsOpt)
+
+ const (
+ fileName = "file.bin"
+ total = 1024 * 1024 // 1 MiB
+ split = 700 * 1024 // where the upload is "backgrounded"
+ )
+
+ // Deterministic non-zero contents so a zeroed hole is easy to spot
+ contents := make([]byte, total)
+ for i := range contents {
+ contents[i] = byte(i%251 + 1)
+ }
+
+ // First connection: write the first part with truncate, as a fresh upload
+ f, err := client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+ require.NoError(t, err)
+ _, err = f.Write(contents[:split])
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+
+ // Resume: re-open WITHOUT truncate and write the remainder at its offset
+ f, err = client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE)
+ require.NoError(t, err)
+ _, err = f.WriteAt(contents[split:], int64(split))
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+
+ // Read the file back and check it is intact - no zeroed prefix
+ rd, err := client.Open(fileName)
+ require.NoError(t, err)
+ got, err := io.ReadAll(rd)
+ require.NoError(t, err)
+ require.NoError(t, rd.Close())
+
+ require.Equal(t, total, len(got), "wrong file size")
+ assert.Equal(t, contents, got, "file contents corrupted on resumed upload")
+}
+
+// Test that opening a file for write with the truncate flag does truncate any
+// existing data, which is the normal overwrite case for most clients.
+func TestFilewriteTruncate(t *testing.T) {
+ vfsOpt := vfscommon.Opt
+ vfsOpt.CacheMode = vfscommon.CacheModeWrites
+ client := startTestServer(t, &vfsOpt)
+
+ const fileName = "file.bin"
+
+ // Write some initial long contents
+ require.NoError(t, writeFile(client, fileName, strings.Repeat("A", 1024)))
+
+ // Overwrite with shorter contents using truncate
+ const newContents = "hello"
+ require.NoError(t, writeFile(client, fileName, newContents))
+
+ rd, err := client.Open(fileName)
+ require.NoError(t, err)
+ got, err := io.ReadAll(rd)
+ require.NoError(t, err)
+ require.NoError(t, rd.Close())
+
+ assert.Equal(t, newContents, string(got))
+}
+
+// writeFile writes contents to fileName via the client truncating any existing
+// data, the way a normal upload does.
+func writeFile(client *sftp.Client, fileName, contents string) error {
+ f, err := client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+ if err != nil {
+ return err
+ }
+ if _, err := f.Write([]byte(contents)); err != nil {
+ _ = f.Close()
+ return err
+ }
+ return f.Close()
+}