Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

swift: add workarounds for bad listings in Ceph RGW

Paul Collinsbd5799c079c0
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index a356557ca..53aab56d4 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -278,6 +278,36 @@ provider.`,
Value: "pca",
Help: "OVH Public Cloud Archive",
}},
+ }, {
+ Name: "fetch_until_empty_page",
+ Help: `When paginating, always fetch unless we received an empty page.
+
+Consider using this option if rclone listings show fewer objects
+than expected, or if repeated syncs copy unchanged objects.
+
+It is safe to enable this, but rclone may make more API calls than
+necessary.
+
+This is one of a pair of workarounds to handle implementations
+of the Swift API that do not implement pagination as expected. See
+also "partial_page_fetch_threshold".`,
+ Default: false,
+ Advanced: true,
+ }, {
+ Name: "partial_page_fetch_threshold",
+ Help: `When paginating, fetch if the current page is within this percentage of the limit.
+
+Consider using this option if rclone listings show fewer objects
+than expected, or if repeated syncs copy unchanged objects.
+
+It is safe to enable this, but rclone may make more API calls than
+necessary.
+
+This is one of a pair of workarounds to handle implementations
+of the Swift API that do not implement pagination as expected. See
+also "fetch_until_empty_page".`,
+ Default: 0,
+ Advanced: true,
}}, SharedOptions...),
})
}
@@ -308,6 +338,8 @@ type Options struct {
NoLargeObjects bool `config:"no_large_objects"`
UseSegmentsContainer fs.Tristate `config:"use_segments_container"`
Enc encoder.MultiEncoder `config:"encoding"`
+ FetchUntilEmptyPage bool `config:"fetch_until_empty_page"`
+ PartialPageFetchThreshold int `config:"partial_page_fetch_threshold"`
}
// Fs represents a remote swift server
@@ -462,6 +494,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
Transport: fshttp.NewTransport(ctx),
+ FetchUntilEmptyPage: opt.FetchUntilEmptyPage,
+ PartialPageFetchThreshold: opt.PartialPageFetchThreshold,
}
if opt.EnvAuth {
err := c.ApplyEnvironment()