Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

archive/squashfs: fix reading images with no fragment or xattr table

maximilize2f3895fa3cd3
diff --git a/backend/archive/archive_internal_test.go b/backend/archive/archive_internal_test.go
index ccbd04ff2..0866b7dec 100644
--- a/backend/archive/archive_internal_test.go
+++ b/backend/archive/archive_internal_test.go
@@ -6,6 +6,7 @@ import (
"bytes"
"context"
"fmt"
+ "io"
"os"
"os/exec"
"path"
@@ -219,3 +220,60 @@ func TestArchiveSquashfs(t *testing.T) {
run(t, "mksquashfs", input, output)
})
}
+
+// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
+// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
+//
+// - 1.sqfs: a single empty directory, so the image has no fragment table
+// (its fragment-table start holds the "not present" sentinel).
+// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
+// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
+// tools-ng can emit. Built by packing a two-file tree with xattrs via
+// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
+// content is trivial placeholder data.
+//
+// Both images used to fail to list. Regression test for #9004.
+func TestArchiveSquashfsIssue9004(t *testing.T) {
+ fstest.Initialise()
+ ctx := context.Background()
+
+ testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
+ require.NoError(t, err)
+
+ archiveFor := func(t *testing.T, name string) fs.Fs {
+ f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
+ require.NoError(t, err)
+ return f
+ }
+
+ t.Run("EmptyDir", func(t *testing.T) {
+ // 1.sqfs is a single empty directory - it must list without error.
+ entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
+ require.NoError(t, err)
+ assert.Equal(t, 0, len(entries))
+ })
+
+ t.Run("NoXattrTree", func(t *testing.T) {
+ f := archiveFor(t, "2.sqfs")
+ entries, err := f.List(ctx, "")
+ require.NoError(t, err)
+ names := make([]string, 0, len(entries))
+ for _, e := range entries {
+ names = append(names, path.Base(e.Remote()))
+ }
+ assert.Contains(t, names, "alpha")
+ assert.Contains(t, names, "beta")
+
+ // A file in the tree must be readable with its real content.
+ obj, err := f.NewObject(ctx, "beta/sample.xml")
+ require.NoError(t, err)
+ assert.Greater(t, obj.Size(), int64(0))
+ rc, err := obj.Open(ctx)
+ require.NoError(t, err)
+ data, err := io.ReadAll(rc)
+ require.NoError(t, err)
+ require.NoError(t, rc.Close())
+ assert.Equal(t, int(obj.Size()), len(data))
+ assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
+ })
+}
diff --git a/backend/archive/squashfs/squashfs.go b/backend/archive/squashfs/squashfs.go
index d04082275..d7c82d8d7 100644
--- a/backend/archive/squashfs/squashfs.go
+++ b/backend/archive/squashfs/squashfs.go
@@ -5,6 +5,7 @@ import (
"context"
"fmt"
"io"
+ "os"
"path"
"strings"
"time"
@@ -180,13 +181,12 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {
}
// Convert a FileInfo into an Object from native dir
-func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
+func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()),
size: item.Size(),
modTime: item.ModTime(),
- item: item,
}
}
@@ -213,26 +213,25 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
entries = make(fs.DirEntries, 0, len(items))
- for _, fi := range items {
- item, ok := fi.(squashfs.FileStat)
- if !ok {
- return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
- }
+ for _, item := range items {
// fs.Debugf(item.Name(), "entry = %#v", item)
var entry fs.DirEntry
- if err != nil {
- return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
- }
if item.IsDir() {
- var remote = f.fromNative(nativeDir, item.Name())
- entry = fs.NewDir(remote, item.ModTime())
- } else {
- if item.Mode().IsRegular() {
- entry = f.objectFromFileInfo(nativeDir, item)
- } else {
- fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
- continue
+ remote := f.fromNative(nativeDir, item.Name())
+ info, err := item.Info()
+ if err != nil {
+ return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
+ }
+ entry = fs.NewDir(remote, info.ModTime())
+ } else if item.Type().IsRegular() {
+ info, err := item.Info()
+ if err != nil {
+ return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
+ entry = f.objectFromFileInfo(nativeDir, info)
+ } else {
+ fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
+ continue
}
entries = append(entries, entry)
}
@@ -260,11 +259,11 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
if fi.IsDir() {
return nil, fs.ErrorNotAFile
}
- item, ok := fi.(squashfs.FileStat)
- if !ok {
- return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
+ info, err := fi.Info()
+ if err != nil {
+ return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
}
- o = f.objectFromFileInfo(dir, item)
+ o = f.objectFromFileInfo(dir, info)
break
}
}
@@ -339,7 +338,6 @@ type Object struct {
remote string
size int64
modTime time.Time
- item squashfs.FileStat
}
// Fs returns read only access to the Fs that this object is part of
@@ -416,8 +414,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
}
fs.Debugf(o, "Opening %q", remote)
- //fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
- fh, err := o.item.Open()
+ fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
if err != nil {
return nil, err
}
diff --git a/backend/archive/squashfs/testdata/1.sqfs b/backend/archive/squashfs/testdata/1.sqfs
new file mode 100644
index 000000000..15e3edb30
Binary files /dev/null and b/backend/archive/squashfs/testdata/1.sqfs differ
diff --git a/backend/archive/squashfs/testdata/2.sqfs b/backend/archive/squashfs/testdata/2.sqfs
new file mode 100644
index 000000000..cf1bd5353
Binary files /dev/null and b/backend/archive/squashfs/testdata/2.sqfs differ
diff --git a/go.mod b/go.mod
index 5ed239307..0edc13e1f 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
github.com/colinmarc/hdfs/v2 v2.4.0
github.com/coreos/go-semver v0.3.1
github.com/coreos/go-systemd/v22 v22.6.0 // v22.7.0 fails to compile on netbsd - don't upgrade until fixed upstream
- github.com/diskfs/go-diskfs v1.9.3
+ github.com/diskfs/go-diskfs v1.9.4
github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0
github.com/gabriel-vasile/mimetype v1.4.13
diff --git a/go.sum b/go.sum
index d416bfe36..4e8871c03 100644
--- a/go.sum
+++ b/go.sum
@@ -182,8 +182,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/diskfs/go-diskfs v1.9.3 h1:cLciNCeZ4QAXVxyPJDr1ZJ9N9CCG3rQlQ/z/Cs/cNDM=
-github.com/diskfs/go-diskfs v1.9.3/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=
+github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=
+github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=