"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
swift: add workarounds for bad listings in Ceph RGW
Paul Collinsbd5799c079c0
diff --git a/backend/swift/swift.go b/backend/swift/swift.goindex a356557ca..53aab56d4 100644--- a/backend/swift/swift.go+++ b/backend/swift/swift.go@@ -278,6 +278,36 @@ provider.`,Value: "pca",Help: "OVH Public Cloud Archive",}},+ }, {+ Name: "fetch_until_empty_page",+ Help: `When paginating, always fetch unless we received an empty page.++Consider using this option if rclone listings show fewer objects+than expected, or if repeated syncs copy unchanged objects.++It is safe to enable this, but rclone may make more API calls than+necessary.++This is one of a pair of workarounds to handle implementations+of the Swift API that do not implement pagination as expected. See+also "partial_page_fetch_threshold".`,+ Default: false,+ Advanced: true,+ }, {+ Name: "partial_page_fetch_threshold",+ Help: `When paginating, fetch if the current page is within this percentage of the limit.++Consider using this option if rclone listings show fewer objects+than expected, or if repeated syncs copy unchanged objects.++It is safe to enable this, but rclone may make more API calls than+necessary.++This is one of a pair of workarounds to handle implementations+of the Swift API that do not implement pagination as expected. See+also "fetch_until_empty_page".`,+ Default: 0,+ Advanced: true,}}, SharedOptions...),})}@@ -308,6 +338,8 @@ type Options struct {NoLargeObjects bool `config:"no_large_objects"`UseSegmentsContainer fs.Tristate `config:"use_segments_container"`Enc encoder.MultiEncoder `config:"encoding"`+ FetchUntilEmptyPage bool `config:"fetch_until_empty_page"`+ PartialPageFetchThreshold int `config:"partial_page_fetch_threshold"`}// Fs represents a remote swift server@@ -462,6 +494,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.ConConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transportTimeout: 10 * ci.Timeout, // Use the timeouts in the transportTransport: fshttp.NewTransport(ctx),+ FetchUntilEmptyPage: opt.FetchUntilEmptyPage,+ PartialPageFetchThreshold: opt.PartialPageFetchThreshold,}if opt.EnvAuth {err := c.ApplyEnvironment()