"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
swift: If storage_policy isn't set, use the root containers policy
Andrew Ruthven04e91838db08
diff --git a/backend/swift/swift.go b/backend/swift/swift.goindex d4d5c5fc9..ac18da01a 100644--- a/backend/swift/swift.go+++ b/backend/swift/swift.go@@ -561,6 +561,21 @@ func (f *Fs) setRoot(root string) {f.rootContainer, f.rootDirectory = bucket.Split(f.root)}+// Fetch the base container's policy to be used if/when we need to create a+// segments container to ensure we use the same policy.+func (f *Fs) fetchStoragePolicy(ctx context.Context, container string) (fs.Fs, error) {+ err := f.pacer.Call(func() (bool, error) {+ var rxHeaders swift.Headers+ _, rxHeaders, err := f.c.Container(ctx, container)++ f.opt.StoragePolicy = rxHeaders["X-Storage-Policy"]+ fs.Debugf(f, "Auto set StoragePolicy to %s", f.opt.StoragePolicy)++ return shouldRetryHeaders(ctx, rxHeaders, err)+ })+ return nil, err+}+// NewFsWithConnection constructs an Fs from the path, container:path// and authenticated connection.//@@ -590,6 +605,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, cf.opt.UseSegmentsContainer.Valid = truefs.Debugf(f, "Auto set use_segments_container to %v", f.opt.UseSegmentsContainer.Value)}+if f.rootContainer != "" && f.rootDirectory != "" {// Check to see if the object exists - ignoring directory markersvar info swift.Object@@ -1132,6 +1148,13 @@ func (f *Fs) newSegmentedUpload(ctx context.Context, dstContainer string, dstPatcontainer: dstContainer,}if f.opt.UseSegmentsContainer.Value {+ if f.opt.StoragePolicy == "" {+ _, err = f.fetchStoragePolicy(ctx, dstContainer)+ if err != nil {+ return nil, err+ }+ }+su.container += segmentsContainerSuffixerr = f.makeContainer(ctx, su.container)if err != nil {diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.goindex f58f8f4b6..9f3d2d74f 100644--- a/backend/swift/swift_test.go+++ b/backend/swift/swift_test.go@@ -76,6 +76,7 @@ func (f *Fs) testNoChunk(t *testing.T) {// Additional tests that aren't in the frameworkfunc (f *Fs) InternalTest(t *testing.T) {+ t.Run("PolicyDiscovery", f.testPolicyDiscovery)t.Run("NoChunk", f.testNoChunk)t.Run("WithChunk", f.testWithChunk)t.Run("WithChunkFail", f.testWithChunkFail)@@ -195,4 +196,50 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {require.Equal(t, obj.Size(), objTarget.Size())}+func (f *Fs) testPolicyDiscovery(t *testing.T) {+ ctx := context.TODO()+ container := "testPolicyDiscovery-1"+ // Reset the policy so we can test if it is populated.+ f.opt.StoragePolicy = ""+ err := f.makeContainer(ctx, container)+ require.NoError(t, err)+ _, err = f.fetchStoragePolicy(ctx, container)+ require.NoError(t, err)++ // Default policy for Swift is Policy-0.+ assert.Equal(t, "Policy-0", f.opt.StoragePolicy)++ // Create a container using a non-default policy, and check to ensure+ // that the created segments container uses the same non-default policy.+ policy := "Policy-1"+ container = "testPolicyDiscovery-2"++ f.opt.StoragePolicy = policy+ err = f.makeContainer(ctx, container)+ require.NoError(t, err)++ // Reset the policy so we can test if it is populated, and set to the+ // non-default policy.+ f.opt.StoragePolicy = ""+ _, err = f.fetchStoragePolicy(ctx, container)+ require.NoError(t, err)+ assert.Equal(t, policy, f.opt.StoragePolicy)++ // Test that when a segmented upload container is made, the newly+ // created container inherits the non-default policy of the base+ // container.+ f.opt.StoragePolicy = ""+ f.opt.UseSegmentsContainer.Value = true+ su, err := f.newSegmentedUpload(ctx, container, "")+ require.NoError(t, err)+ // The container name we expected?+ segmentsContainer := container + segmentsContainerSuffix+ assert.Equal(t, segmentsContainer, su.container)+ // The policy we expected?+ f.opt.StoragePolicy = ""+ _, err = f.fetchStoragePolicy(ctx, su.container)+ require.NoError(t, err)+ assert.Equal(t, policy, f.opt.StoragePolicy)+}+var _ fstests.InternalTester = (*Fs)(nil)diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIOindex 7a04d63ff..d46ad5029 100755--- a/fstest/testserver/init.d/TestSwiftAIO+++ b/fstest/testserver/init.d/TestSwiftAIO@@ -8,10 +8,12 @@ PORT=28628. $(dirname "$0")/docker.bashstart() {+ # We need to replace the remakerings in the container to create Policy-1.docker run --rm -d --name ${NAME} \-p 127.0.0.1:${PORT}:8080 \+ -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/swift/bin/remakerings:ro \bouncestorage/swift-aio-+echo type=swiftecho env_auth=falseecho user=test:testerdiff --git a/fstest/testserver/init.d/TestSwiftAIO.d/remakerings b/fstest/testserver/init.d/TestSwiftAIO.d/remakeringsnew file mode 100755index 000000000..06785b3ab--- /dev/null+++ b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings@@ -0,0 +1,33 @@+#!/bin/bash++cd /etc/swift++if ! grep storage-policy swift.conf; then+ cat <<EOF >> swift.conf++# Policy-0 is the default, we need two policies to test policy inheritance.+[storage-policy:0]+name = Policy-0+default = true++[storage-policy:1]+name = Policy-1+EOF+fi++rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz++swift-ring-builder object.builder create 10 1 1+swift-ring-builder object.builder add r1z1-127.0.0.1:6010/sdb1 1+swift-ring-builder object.builder rebalance+swift-ring-builder container.builder create 10 1 1+swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1+swift-ring-builder container.builder rebalance+swift-ring-builder account.builder create 10 1 1+swift-ring-builder account.builder add r1z1-127.0.0.1:6012/sdb1 1+swift-ring-builder account.builder rebalance++# For Policy-1:+swift-ring-builder object-1.builder create 10 1 1+swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1+swift-ring-builder object-1.builder rebalance