"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs/march: fix runtime: program exceeds 10000-thread limit
Nick Craig-Wood07e76419c922
diff --git a/fs/march/march.go b/fs/march/march.goindex 8b19baa6e..9123f8a7e 100644--- a/fs/march/march.go+++ b/fs/march/march.go@@ -16,6 +16,7 @@ import ("github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/transform"+ "golang.org/x/sync/semaphore""golang.org/x/text/unicode/norm")@@ -41,9 +42,10 @@ type March struct {NoCheckDest bool // transfer all objects regardless without checking dstNoUnicodeNormalization bool // don't normalize unicode characters in filenames// internal state- srcListDir listDirFn // function to call to list a directory in the src- dstListDir listDirFn // function to call to list a directory in the dst- transforms []matchTransformFn+ srcListDir listDirFn // function to call to list a directory in the src+ dstListDir listDirFn // function to call to list a directory in the dst+ transforms []matchTransformFn+ newObjectSem *semaphore.Weighted // make sure we don't call too many NewObjects simultaneously}// Marcher is called on each match@@ -78,6 +80,8 @@ func (m *March) init(ctx context.Context) {if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {m.transforms = append(m.transforms, strings.ToLower)}+ // Only allow ci.Checkers simultaneous calls to NewObject+ m.newObjectSem = semaphore.NewWeighted(int64(ci.Checkers))}// srcOrDstKey turns a directory entry into a sort key using the defined transforms.@@ -461,7 +465,12 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {continue}leaf := path.Base(t.src.Remote())+ if err := m.newObjectSem.Acquire(m.Ctx, 1); err != nil {+ t.dstMatch <- nil+ continue+ }dst, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))+ m.newObjectSem.Release(1)if err != nil {dst = nil}