"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
list: fix nil pointer panic in Sorter when temp file creation fails
Nick Craig-Wood5d6690eb20f4
diff --git a/fs/list/sorter.go b/fs/list/sorter.goindex b2674b294..cfe821238 100644--- a/fs/list/sorter.go+++ b/fs/list/sorter.go@@ -50,6 +50,7 @@ type Sorter struct {errChan <-chan error // for getting errors from the ext sortsorter *extsort.StringSorter // external string sorterrs *errcount.ErrCount // accumulate errors+ tempDir string // directory for temp files (empty for default)}// KeyFn turns an entry into a sort key@@ -144,6 +145,7 @@ func (ls *Sorter) startExtSort() (err error) {ChanBuffSize: 1024, // small effectSortedChanBuffSize: 1024, // makes a lot of differenceChunkSize: 32 * 1024, // tuned for 50 char records (UUID sized)+ TempFilesDir: ls.tempDir,// Defaults// ChunkSize: int(1e6), // amount of records to store in each chunk which will be written to disk// NumWorkers: 2, // maximum number of workers to use for parallel sorting@@ -152,6 +154,17 @@ func (ls *Sorter) startExtSort() (err error) {// TempFilesDir: "", // empty for use OS default ex: /tmp}ls.sorter, ls.outputChan, ls.errChan = extsort.Strings(ls.inputChan, &opt)+ if ls.sorter == nil {+ // extsort.Strings returns nil when it can't create temp+ // files (e.g. due to permissions or apparmor restrictions).+ // The error will be on errChan.+ select {+ case err = <-ls.errChan:+ default:+ err = errors.New("failed to initialise on-disk sort")+ }+ return fmt.Errorf("sorter: %w", err)+ }go ls.sorter.Sort(ls.ctx)// Show we are extsorting nowdiff --git a/fs/list/sorter_test.go b/fs/list/sorter_test.goindex b8db3725e..b1941c481 100644--- a/fs/list/sorter_test.go+++ b/fs/list/sorter_test.go@@ -4,6 +4,9 @@ import ("cmp""context""fmt"+ "os"+ "path/filepath"+ "runtime""slices""strings""testing"@@ -231,6 +234,51 @@ func TestSorterExt(t *testing.T) {}}+// Test that startExtSort returns an error instead of panicking when+// temp file creation fails (e.g. due to permissions or apparmor).+// See: https://github.com/rclone/rclone/issues/9244+func TestSorterExtTempFileError(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ ci.ListCutoff = 1 // force ext sort on first Add++ callback := func(entries fs.DirEntries) error {+ return nil+ }+ ls, err := NewSorter(ctx, nil, callback, nil)+ require.NoError(t, err)+ defer ls.CleanUp()++ // Override tempDir to a path where os.MkdirAll will fail,+ // simulating apparmor or permission denial. This makes+ // extsort.Strings return a nil sorter.+ if runtime.GOOS == "windows" {+ // On Windows, create a regular file and use a child path.+ // os.MkdirAll fails because an intermediate component is a file.+ // Windows stat returns ERROR_PATH_NOT_FOUND (treated as+ // IsNotExist) so the extsort library won't fall back to a+ // default directory.+ blocker := filepath.Join(t.TempDir(), "notadir")+ require.NoError(t, os.WriteFile(blocker, []byte("x"), 0644))+ ls.tempDir = filepath.Join(blocker, "sub")+ } else {+ // On Unix, create a read-only parent directory. stat on the+ // child returns ENOENT so the extsort library won't fall+ // back, but os.MkdirAll fails with EACCES.+ parent := filepath.Join(t.TempDir(), "roparent")+ require.NoError(t, os.Mkdir(parent, 0o755))+ require.NoError(t, os.Chmod(parent, 0o555))+ t.Cleanup(func() { _ = os.Chmod(parent, 0o755) })+ ls.tempDir = filepath.Join(parent, "child")+ }++ // Add enough entries to trigger ext sort. Before the fix this+ // would panic with nil pointer dereference.+ err = ls.Add(fs.DirEntries{mockobject.Object("a"), mockobject.Object("b")})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "sorter:")+}+// benchFs implements enough of the fs.Fs interface for Sortertype benchFs struct{}