Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

swift: If storage_policy isn't set, use the root containers policy

Andrew Ruthven04e91838db08
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index d4d5c5fc9..ac18da01a 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -561,6 +561,21 @@ func (f *Fs) setRoot(root string) {
f.rootContainer, f.rootDirectory = bucket.Split(f.root)
}
+// Fetch the base container's policy to be used if/when we need to create a
+// segments container to ensure we use the same policy.
+func (f *Fs) fetchStoragePolicy(ctx context.Context, container string) (fs.Fs, error) {
+ err := f.pacer.Call(func() (bool, error) {
+ var rxHeaders swift.Headers
+ _, rxHeaders, err := f.c.Container(ctx, container)
+
+ f.opt.StoragePolicy = rxHeaders["X-Storage-Policy"]
+ fs.Debugf(f, "Auto set StoragePolicy to %s", f.opt.StoragePolicy)
+
+ return shouldRetryHeaders(ctx, rxHeaders, err)
+ })
+ return nil, err
+}
+
// NewFsWithConnection constructs an Fs from the path, container:path
// and authenticated connection.
//
@@ -590,6 +605,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c
f.opt.UseSegmentsContainer.Valid = true
fs.Debugf(f, "Auto set use_segments_container to %v", f.opt.UseSegmentsContainer.Value)
}
+
if f.rootContainer != "" && f.rootDirectory != "" {
// Check to see if the object exists - ignoring directory markers
var info swift.Object
@@ -1132,6 +1148,13 @@ func (f *Fs) newSegmentedUpload(ctx context.Context, dstContainer string, dstPat
container: dstContainer,
}
if f.opt.UseSegmentsContainer.Value {
+ if f.opt.StoragePolicy == "" {
+ _, err = f.fetchStoragePolicy(ctx, dstContainer)
+ if err != nil {
+ return nil, err
+ }
+ }
+
su.container += segmentsContainerSuffix
err = f.makeContainer(ctx, su.container)
if err != nil {
diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.go
index f58f8f4b6..9f3d2d74f 100644
--- a/backend/swift/swift_test.go
+++ b/backend/swift/swift_test.go
@@ -76,6 +76,7 @@ func (f *Fs) testNoChunk(t *testing.T) {
// Additional tests that aren't in the framework
func (f *Fs) InternalTest(t *testing.T) {
+ t.Run("PolicyDiscovery", f.testPolicyDiscovery)
t.Run("NoChunk", f.testNoChunk)
t.Run("WithChunk", f.testWithChunk)
t.Run("WithChunkFail", f.testWithChunkFail)
@@ -195,4 +196,50 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
require.Equal(t, obj.Size(), objTarget.Size())
}
+func (f *Fs) testPolicyDiscovery(t *testing.T) {
+ ctx := context.TODO()
+ container := "testPolicyDiscovery-1"
+ // Reset the policy so we can test if it is populated.
+ f.opt.StoragePolicy = ""
+ err := f.makeContainer(ctx, container)
+ require.NoError(t, err)
+ _, err = f.fetchStoragePolicy(ctx, container)
+ require.NoError(t, err)
+
+ // Default policy for Swift is Policy-0.
+ assert.Equal(t, "Policy-0", f.opt.StoragePolicy)
+
+ // Create a container using a non-default policy, and check to ensure
+ // that the created segments container uses the same non-default policy.
+ policy := "Policy-1"
+ container = "testPolicyDiscovery-2"
+
+ f.opt.StoragePolicy = policy
+ err = f.makeContainer(ctx, container)
+ require.NoError(t, err)
+
+ // Reset the policy so we can test if it is populated, and set to the
+ // non-default policy.
+ f.opt.StoragePolicy = ""
+ _, err = f.fetchStoragePolicy(ctx, container)
+ require.NoError(t, err)
+ assert.Equal(t, policy, f.opt.StoragePolicy)
+
+ // Test that when a segmented upload container is made, the newly
+ // created container inherits the non-default policy of the base
+ // container.
+ f.opt.StoragePolicy = ""
+ f.opt.UseSegmentsContainer.Value = true
+ su, err := f.newSegmentedUpload(ctx, container, "")
+ require.NoError(t, err)
+ // The container name we expected?
+ segmentsContainer := container + segmentsContainerSuffix
+ assert.Equal(t, segmentsContainer, su.container)
+ // The policy we expected?
+ f.opt.StoragePolicy = ""
+ _, err = f.fetchStoragePolicy(ctx, su.container)
+ require.NoError(t, err)
+ assert.Equal(t, policy, f.opt.StoragePolicy)
+}
+
var _ fstests.InternalTester = (*Fs)(nil)
diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO
index 7a04d63ff..d46ad5029 100755
--- a/fstest/testserver/init.d/TestSwiftAIO
+++ b/fstest/testserver/init.d/TestSwiftAIO
@@ -8,10 +8,12 @@ PORT=28628
. $(dirname "$0")/docker.bash
start() {
+ # We need to replace the remakerings in the container to create Policy-1.
docker run --rm -d --name ${NAME} \
-p 127.0.0.1:${PORT}:8080 \
+ -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/swift/bin/remakerings:ro \
bouncestorage/swift-aio
-
+
echo type=swift
echo env_auth=false
echo user=test:tester
diff --git a/fstest/testserver/init.d/TestSwiftAIO.d/remakerings b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings
new file mode 100755
index 000000000..06785b3ab
--- /dev/null
+++ b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+cd /etc/swift
+
+if ! grep storage-policy swift.conf; then
+ cat <<EOF >> swift.conf
+
+# Policy-0 is the default, we need two policies to test policy inheritance.
+[storage-policy:0]
+name = Policy-0
+default = true
+
+[storage-policy:1]
+name = Policy-1
+EOF
+fi
+
+rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz
+
+swift-ring-builder object.builder create 10 1 1
+swift-ring-builder object.builder add r1z1-127.0.0.1:6010/sdb1 1
+swift-ring-builder object.builder rebalance
+swift-ring-builder container.builder create 10 1 1
+swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1
+swift-ring-builder container.builder rebalance
+swift-ring-builder account.builder create 10 1 1
+swift-ring-builder account.builder add r1z1-127.0.0.1:6012/sdb1 1
+swift-ring-builder account.builder rebalance
+
+# For Policy-1:
+swift-ring-builder object-1.builder create 10 1 1
+swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1
+swift-ring-builder object-1.builder rebalance