Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

swift: Report disk usage in segment containers

Andrew Ruthven5420dbbe38bc
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index ac18da01a..cf4ce9d37 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -943,6 +943,20 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
used = container.Bytes
objects = container.Count
total = container.QuotaBytes
+
+ if f.opt.UseSegmentsContainer.Value {
+ err = f.pacer.Call(func() (bool, error) {
+ segmentsContainer := f.rootContainer + segmentsContainerSuffix
+ container, _, err = f.c.Container(ctx, segmentsContainer)
+ return shouldRetry(ctx, err)
+ })
+ if err != nil && err != swift.ContainerNotFound {
+ return nil, fmt.Errorf("container info failed: %w", err)
+ }
+ if err == nil {
+ used += container.Bytes
+ }
+ }
} else {
var containers []swift.Container
err = f.pacer.Call(func() (bool, error) {
diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.go
index d33e855b4..474df389f 100644
--- a/backend/swift/swift_test.go
+++ b/backend/swift/swift_test.go
@@ -56,6 +56,11 @@ func (f *Fs) testNoChunk(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
+ // Track how much space is used before we put our object.
+ usage, err := f.About(ctx)
+ require.NoError(t, err)
+ usedBeforePut := *usage.Used
+
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
obj, err := f.Features().PutStream(ctx, in, obji)
@@ -70,6 +75,13 @@ func (f *Fs) testNoChunk(t *testing.T) {
require.NoError(t, err)
file.Check(t, obj, f.Precision())
+ // Check how much space is used after the upload, should match the amount we
+ // uploaded..
+ usage, err = f.About(ctx)
+ require.NoError(t, err)
+ expectedUsed := usedBeforePut + obj.Size()
+ require.EqualValues(t, expectedUsed, *usage.Used)
+
// Delete the object
assert.NoError(t, obj.Remove(ctx))
}
@@ -105,12 +117,24 @@ func (f *Fs) testWithChunk(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
+ // Track how much space is used before we put our object.
+ ctx := context.TODO()
+ usage, err := f.About(ctx)
+ require.NoError(t, err)
+ usedBeforePut := *usage.Used
+
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
- ctx := context.TODO()
obj, err := f.Features().PutStream(ctx, in, obji)
require.NoError(t, err)
require.NotEmpty(t, obj)
+
+ // Check how much space is used after the upload, should match the amount we
+ // uploaded..
+ usage, err = f.About(ctx)
+ require.NoError(t, err)
+ expectedUsed := usedBeforePut + obj.Size()
+ require.EqualValues(t, expectedUsed, *usage.Used)
}
func (f *Fs) testWithChunkFail(t *testing.T) {
@@ -183,9 +207,14 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
+ // Track how much space is used before we put our object.
+ ctx := context.TODO()
+ usage, err := f.About(ctx)
+ require.NoError(t, err)
+ usedBeforePut := *usage.Used
+
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
- ctx := context.TODO()
obj, err := f.Features().PutStream(ctx, in, obji)
require.NoError(t, err)
require.NotEmpty(t, obj)
@@ -194,6 +223,13 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, objTarget)
require.Equal(t, obj.Size(), objTarget.Size())
+
+ // Check how much space is used after the upload, should match the amount we
+ // uploaded *and* the copy.
+ usage, err = f.About(ctx)
+ require.NoError(t, err)
+ expectedUsed := usedBeforePut + obj.Size() + objTarget.Size()
+ require.EqualValues(t, expectedUsed, *usage.Used)
}
func (f *Fs) testPolicyDiscovery(t *testing.T) {