Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

b2: Add tests for new `cleanup` and `cleanup-hidden` backend commands.

Pat Patterson93c960df5949
diff --git a/backend/b2/b2_internal_test.go b/backend/b2/b2_internal_test.go
index 8253ca167..b922301b1 100644
--- a/backend/b2/b2_internal_test.go
+++ b/backend/b2/b2_internal_test.go
@@ -1,11 +1,25 @@
package b2
import (
+ "context"
+ "crypto/sha1"
+ "fmt"
+ "path"
+ "strings"
"testing"
"time"
+ "github.com/rclone/rclone/backend/b2/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
+ "github.com/rclone/rclone/lib/bucket"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/lib/version"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
// Test b2 string encoding
@@ -170,9 +184,234 @@ func TestParseTimeString(t *testing.T) {
}
+// This is adapted from the s3 equivalent.
+func (f *Fs) InternalTestMetadata(t *testing.T) {
+ ctx := context.Background()
+ original := random.String(1000)
+ contents := fstest.Gz(t, original)
+ mimeType := "text/html"
+
+ item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
+ btime := time.Now()
+ obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, mimeType, nil)
+ defer func() {
+ assert.NoError(t, obj.Remove(ctx))
+ }()
+ o := obj.(*Object)
+ gotMetadata, err := o.getMetaData(ctx)
+ require.NoError(t, err)
+
+ // We currently have a limited amount of metadata to test with B2
+ assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
+
+ // Modification time from the x-bz-info-src_last_modified_millis header
+ var mtime api.Timestamp
+ err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
+ if err != nil {
+ fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
+ }
+ assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
+
+ // Upload time
+ gotBtime := time.Time(gotMetadata.UploadTimestamp)
+ dt := gotBtime.Sub(btime)
+ assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
+
+ t.Run("GzipEncoding", func(t *testing.T) {
+ // Test that the gzipped file we uploaded can be
+ // downloaded
+ checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+ gotContents := fstests.ReadObject(ctx, t, o, -1)
+ assert.Equal(t, wantContents, gotContents)
+ assert.Equal(t, wantSize, o.Size())
+ gotHash, err := o.Hash(ctx, hash.SHA1)
+ require.NoError(t, err)
+ assert.Equal(t, wantHash, gotHash)
+ }
+
+ t.Run("NoDecompress", func(t *testing.T) {
+ checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
+ })
+ })
+}
+
+func sha1Sum(t *testing.T, s string) string {
+ hash := sha1.Sum([]byte(s))
+ return fmt.Sprintf("%x", hash)
+}
+
+// This is adapted from the s3 equivalent.
+func (f *Fs) InternalTestVersions(t *testing.T) {
+ ctx := context.Background()
+
+ // Small pause to make the LastModified different since AWS
+ // only seems to track them to 1 second granularity
+ time.Sleep(2 * time.Second)
+
+ // Create an object
+ const dirName = "versions"
+ const fileName = dirName + "/" + "test-versions.txt"
+ contents := random.String(100)
+ item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+ defer func() {
+ assert.NoError(t, obj.Remove(ctx))
+ }()
+ objMetadata, err := obj.(*Object).getMetaData(ctx)
+ require.NoError(t, err)
+
+ // Small pause
+ time.Sleep(2 * time.Second)
+
+ // Remove it
+ assert.NoError(t, obj.Remove(ctx))
+
+ // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
+ time.Sleep(2 * time.Second)
+
+ // And create it with different size and contents
+ newContents := random.String(101)
+ newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
+ newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
+ newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
+ require.NoError(t, err)
+
+ t.Run("Versions", func(t *testing.T) {
+ // Set --b2-versions for this test
+ f.opt.Versions = true
+ defer func() {
+ f.opt.Versions = false
+ }()
+
+ // Read the contents
+ entries, err := f.List(ctx, dirName)
+ require.NoError(t, err)
+ tests := 0
+ var fileNameVersion string
+ for _, entry := range entries {
+ t.Log(entry)
+ remote := entry.Remote()
+ if remote == fileName {
+ t.Run("ReadCurrent", func(t *testing.T) {
+ assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+ })
+ tests++
+ } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
+ t.Run("ReadVersion", func(t *testing.T) {
+ assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+ })
+ assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
+ fileNameVersion = remote
+ tests++
+ }
+ }
+ assert.Equal(t, 2, tests, "object missing from listing")
+
+ // Check we can read the object with a version suffix
+ t.Run("NewObject", func(t *testing.T) {
+ o, err := f.NewObject(ctx, fileNameVersion)
+ require.NoError(t, err)
+ require.NotNil(t, o)
+ assert.Equal(t, int64(100), o.Size(), o.Remote())
+ })
+
+ // Check we can make a NewFs from that object with a version suffix
+ t.Run("NewFs", func(t *testing.T) {
+ newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
+ // Make sure --b2-versions is set in the config of the new remote
+ fs.Debugf(nil, "oldPath = %q", newPath)
+ lastColon := strings.LastIndex(newPath, ":")
+ require.True(t, lastColon >= 0)
+ newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
+ fs.Debugf(nil, "newPath = %q", newPath)
+ fNew, err := cache.Get(ctx, newPath)
+ // This should return pointing to a file
+ require.Equal(t, fs.ErrorIsFile, err)
+ require.NotNil(t, fNew)
+ // With the directory above
+ assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
+ })
+ })
+
+ t.Run("VersionAt", func(t *testing.T) {
+ // We set --b2-version-at for this test so make sure we reset it at the end
+ defer func() {
+ f.opt.VersionAt = fs.Time{}
+ }()
+
+ var (
+ firstObjectTime = time.Time(objMetadata.UploadTimestamp)
+ secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
+ )
+
+ for _, test := range []struct {
+ what string
+ at time.Time
+ want []fstest.Item
+ wantErr error
+ wantSize int64
+ }{
+ {
+ what: "Before",
+ at: firstObjectTime.Add(-time.Second),
+ want: fstests.InternalTestFiles,
+ wantErr: fs.ErrorObjectNotFound,
+ },
+ {
+ what: "AfterOne",
+ at: firstObjectTime.Add(time.Second),
+ want: append([]fstest.Item{item}, fstests.InternalTestFiles...),
+ wantSize: 100,
+ },
+ {
+ what: "AfterDelete",
+ at: secondObjectTime.Add(-time.Second),
+ want: fstests.InternalTestFiles,
+ wantErr: fs.ErrorObjectNotFound,
+ },
+ {
+ what: "AfterTwo",
+ at: secondObjectTime.Add(time.Second),
+ want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
+ wantSize: 101,
+ },
+ } {
+ t.Run(test.what, func(t *testing.T) {
+ f.opt.VersionAt = fs.Time(test.at)
+ t.Run("List", func(t *testing.T) {
+ fstest.CheckListing(t, f, test.want)
+ })
+ // b2 NewObject doesn't work with VersionAt
+ //t.Run("NewObject", func(t *testing.T) {
+ // gotObj, gotErr := f.NewObject(ctx, fileName)
+ // assert.Equal(t, test.wantErr, gotErr)
+ // if gotErr == nil {
+ // assert.Equal(t, test.wantSize, gotObj.Size())
+ // }
+ //})
+ })
+ }
+ })
+
+ t.Run("Cleanup", func(t *testing.T) {
+ require.NoError(t, f.cleanUp(ctx, true, false, 0))
+ items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+ fstest.CheckListing(t, f, items)
+ // Set --b2-versions for this test
+ f.opt.Versions = true
+ defer func() {
+ f.opt.Versions = false
+ }()
+ fstest.CheckListing(t, f, items)
+ })
+
+ // Purge gets tested later
+}
+
// -run TestIntegration/FsMkdir/FsPutFiles/Internal
func (f *Fs) InternalTest(t *testing.T) {
- // Internal tests go here
+ t.Run("Metadata", f.InternalTestMetadata)
+ t.Run("Versions", f.InternalTestVersions)
}
var _ fstests.InternalTester = (*Fs)(nil)
diff --git a/fstest/fstest.go b/fstest/fstest.go
index ebea012c7..835a76d60 100644
--- a/fstest/fstest.go
+++ b/fstest/fstest.go
@@ -5,6 +5,7 @@ package fstest
import (
"bytes"
+ "compress/gzip"
"context"
"flag"
"fmt"
@@ -609,3 +610,14 @@ func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Director
gotT := dir.ModTime(ctx)
AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision())
}
+
+// Gz returns a compressed version of its input string
+func Gz(t *testing.T, s string) string {
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ _, err := zw.Write([]byte(s))
+ require.NoError(t, err)
+ err = zw.Close()
+ require.NoError(t, err)
+ return buf.String()
+}