"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
archive/squashfs: fix reading images with no fragment or xattr table
maximilize2f3895fa3cd3
diff --git a/backend/archive/archive_internal_test.go b/backend/archive/archive_internal_test.goindex ccbd04ff2..0866b7dec 100644--- a/backend/archive/archive_internal_test.go+++ b/backend/archive/archive_internal_test.go@@ -6,6 +6,7 @@ import ("bytes""context""fmt"+ "io""os""os/exec""path"@@ -219,3 +220,60 @@ func TestArchiveSquashfs(t *testing.T) {run(t, "mksquashfs", input, output)})}++// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise+// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):+//+// - 1.sqfs: a single empty directory, so the image has no fragment table+// (its fragment-table start holds the "not present" sentinel).+// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while+// inodes still carry a (non-sentinel) xattr index - the shape squashfs-+// tools-ng can emit. Built by packing a two-file tree with xattrs via+// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree+// content is trivial placeholder data.+//+// Both images used to fail to list. Regression test for #9004.+func TestArchiveSquashfsIssue9004(t *testing.T) {+ fstest.Initialise()+ ctx := context.Background()++ testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))+ require.NoError(t, err)++ archiveFor := func(t *testing.T, name string) fs.Fs {+ f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))+ require.NoError(t, err)+ return f+ }++ t.Run("EmptyDir", func(t *testing.T) {+ // 1.sqfs is a single empty directory - it must list without error.+ entries, err := archiveFor(t, "1.sqfs").List(ctx, "")+ require.NoError(t, err)+ assert.Equal(t, 0, len(entries))+ })++ t.Run("NoXattrTree", func(t *testing.T) {+ f := archiveFor(t, "2.sqfs")+ entries, err := f.List(ctx, "")+ require.NoError(t, err)+ names := make([]string, 0, len(entries))+ for _, e := range entries {+ names = append(names, path.Base(e.Remote()))+ }+ assert.Contains(t, names, "alpha")+ assert.Contains(t, names, "beta")++ // A file in the tree must be readable with its real content.+ obj, err := f.NewObject(ctx, "beta/sample.xml")+ require.NoError(t, err)+ assert.Greater(t, obj.Size(), int64(0))+ rc, err := obj.Open(ctx)+ require.NoError(t, err)+ data, err := io.ReadAll(rc)+ require.NoError(t, err)+ require.NoError(t, rc.Close())+ assert.Equal(t, int(obj.Size()), len(data))+ assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))+ })+}diff --git a/backend/archive/squashfs/squashfs.go b/backend/archive/squashfs/squashfs.goindex d04082275..d7c82d8d7 100644--- a/backend/archive/squashfs/squashfs.go+++ b/backend/archive/squashfs/squashfs.go@@ -5,6 +5,7 @@ import ("context""fmt""io"+ "os""path""strings""time"@@ -180,13 +181,12 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {}// Convert a FileInfo into an Object from native dir-func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {+func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {return &Object{fs: f,remote: f.fromNative(nativeDir, item.Name()),size: item.Size(),modTime: item.ModTime(),- item: item,}}@@ -213,26 +213,25 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e}entries = make(fs.DirEntries, 0, len(items))- for _, fi := range items {- item, ok := fi.(squashfs.FileStat)- if !ok {- return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)- }+ for _, item := range items {// fs.Debugf(item.Name(), "entry = %#v", item)var entry fs.DirEntry- if err != nil {- return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)- }if item.IsDir() {- var remote = f.fromNative(nativeDir, item.Name())- entry = fs.NewDir(remote, item.ModTime())- } else {- if item.Mode().IsRegular() {- entry = f.objectFromFileInfo(nativeDir, item)- } else {- fs.Debugf(item.Name(), "FIXME Not regular file - skipping")- continue+ remote := f.fromNative(nativeDir, item.Name())+ info, err := item.Info()+ if err != nil {+ return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)+ }+ entry = fs.NewDir(remote, info.ModTime())+ } else if item.Type().IsRegular() {+ info, err := item.Info()+ if err != nil {+ return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)}+ entry = f.objectFromFileInfo(nativeDir, info)+ } else {+ fs.Debugf(item.Name(), "FIXME Not regular file - skipping")+ continue}entries = append(entries, entry)}@@ -260,11 +259,11 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {if fi.IsDir() {return nil, fs.ErrorNotAFile}- item, ok := fi.(squashfs.FileStat)- if !ok {- return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)+ info, err := fi.Info()+ if err != nil {+ return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)}- o = f.objectFromFileInfo(dir, item)+ o = f.objectFromFileInfo(dir, info)break}}@@ -339,7 +338,6 @@ type Object struct {remote stringsize int64modTime time.Time- item squashfs.FileStat}// Fs returns read only access to the Fs that this object is part of@@ -416,8 +414,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read}fs.Debugf(o, "Opening %q", remote)- //fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)- fh, err := o.item.Open()+ fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)if err != nil {return nil, err}diff --git a/backend/archive/squashfs/testdata/1.sqfs b/backend/archive/squashfs/testdata/1.sqfsnew file mode 100644index 000000000..15e3edb30Binary files /dev/null and b/backend/archive/squashfs/testdata/1.sqfs differdiff --git a/backend/archive/squashfs/testdata/2.sqfs b/backend/archive/squashfs/testdata/2.sqfsnew file mode 100644index 000000000..cf1bd5353Binary files /dev/null and b/backend/archive/squashfs/testdata/2.sqfs differdiff --git a/go.mod b/go.modindex 5ed239307..0edc13e1f 100644--- a/go.mod+++ b/go.mod@@ -35,7 +35,7 @@ require (github.com/colinmarc/hdfs/v2 v2.4.0github.com/coreos/go-semver v0.3.1github.com/coreos/go-systemd/v22 v22.6.0 // v22.7.0 fails to compile on netbsd - don't upgrade until fixed upstream- github.com/diskfs/go-diskfs v1.9.3+ github.com/diskfs/go-diskfs v1.9.4github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0github.com/gabriel-vasile/mimetype v1.4.13diff --git a/go.sum b/go.sumindex d416bfe36..4e8871c03 100644--- a/go.sum+++ b/go.sum@@ -182,8 +182,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsgithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=-github.com/diskfs/go-diskfs v1.9.3 h1:cLciNCeZ4QAXVxyPJDr1ZJ9N9CCG3rQlQ/z/Cs/cNDM=-github.com/diskfs/go-diskfs v1.9.3/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=+github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=+github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=