"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
// Package squashfs implements a squashfs archiver for the archive backend
package squashfs
import (
"context"
"fmt"
"io"
"os"
"path"
"strings"
"time"
"github.com/diskfs/go-diskfs/filesystem/squashfs"
"github.com/rclone/rclone/backend/archive/archiver"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/sanitize"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
func init() {
archiver.Register(archiver.Archiver{
New: New,
Extension: ".sqfs",
})
}
// Fs represents a wrapped fs.Fs
type Fs struct {
f fs.Fs
wrapper fs.Fs
name string
features *fs.Features // optional features
vfs *vfs.VFS
sqfs *squashfs.FileSystem // interface to the squashfs
c *cache
node vfs.Node // squashfs file object - set if reading
remote string // remote of the squashfs file object
prefix string // position for objects
prefixSlash string // position for objects with a slash on
root string // position to read from within the archive
file string // remote of the single file root points at, if any
links bool // read symlinks as .rclonelink files
}
// recoverParsePanic converts a panic raised while parsing a squashfs image
// into an error assigned through err.
func recoverParsePanic(err *error) {
if r := recover(); r != nil {
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
}
}
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
// panics raised while reading become errors.
//
// The parser reads file data lazily, so an image which opens cleanly can
// still panic on the first Read.
type recoveringReadCloser struct {
io.ReadCloser
}
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
defer recoverParsePanic(&err)
return r.ReadCloser.Read(p)
}
// New constructs an Fs from the (wrappedFs, remote) with the objects
// prefix with prefix and rooted at root
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
defer recoverParsePanic(&err)
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
}
c := newCache(node)
// FIXME blocksize
sqfs, err := squashfs.Read(c, node.Size(), 0, 1024*1024)
if err != nil {
return nil, fmt.Errorf("failed to read squashfs: %w", err)
}
f := &Fs{
f: wrappedFs,
name: path.Join(fs.ConfigString(wrappedFs), remote),
vfs: VFS,
node: node,
sqfs: sqfs,
c: c,
remote: remote,
root: strings.Trim(root, "/"),
links: fs.GetConfig(ctx).Links,
prefix: prefix,
prefixSlash: prefix + "/",
}
if prefix == "" {
f.prefixSlash = ""
}
singleObject := false
// Find the directory the root points to
if f.root != "" && !strings.HasSuffix(root, "/") {
native, err := f.toNative("")
if err == nil {
native = strings.TrimRight(native, "/")
_, err := f.newObjectNative(native)
if err == nil {
// If it pointed to a file, find the directory above
// and remember the file so that only it is listed
f.root = path.Dir(f.root)
if f.root == "." || f.root == "/" {
f.root = ""
}
f.file = f.prefixSlash + path.Base(native)
singleObject = true
}
}
}
// FIXME
// the features here are ones we could support, and they are
// ANDed with the ones from wrappedFs
//
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
f.features = (&fs.Features{
CaseInsensitive: false,
DuplicateFiles: false,
ReadMimeType: false, // MimeTypes not supported with gsquashfs
WriteMimeType: false,
BucketBased: false,
CanHaveEmptyDirectories: true,
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
if singleObject {
return f, fs.ErrorIsFile
}
return f, nil
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
// Include the file so the fs cache can tell this Fs apart from
// the one for the directory containing it
return path.Join(f.root, f.file)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// String returns a description of the FS
func (f *Fs) String() string {
return fmt.Sprintf("Squashfs %q", f.name)
}
// This turns a remote into a native path in the squashfs starting with a /
func (f *Fs) toNative(remote string) (string, error) {
native := strings.Trim(remote, "/")
if f.prefix == "" {
native = "/" + native
} else if native == f.prefix {
native = "/"
} else if !strings.HasPrefix(native, f.prefixSlash) {
return "", fmt.Errorf("internal error: %q doesn't start with prefix %q", native, f.prefixSlash)
} else {
native = native[len(f.prefix):]
}
if f.root != "" {
native = "/" + f.root + native
}
return native, nil
}
// This turns a native path with a leading / into a path suitable for
// the squashfs library which requires io/fs.ValidPath paths with no
// leading / and "." for the root directory.
func toIOFS(native string) string {
native = strings.Trim(native, "/")
if native == "" {
return "."
}
return native
}
// Turn a (nativeDir, leaf) into a remote
func (f *Fs) fromNative(nativeDir string, leaf string) string {
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
dir := nativeDir
if f.root != "" {
dir = strings.TrimPrefix(dir, "/"+f.root)
}
remote := f.prefixSlash + strings.Trim(path.Join(dir, leaf), "/")
// fs.Debugf(nil, "dir = %q, remote=%q", dir, remote)
return remote
}
// Convert a FileInfo into an Object from native dir
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()),
size: item.Size(),
modTime: item.ModTime(),
}
}
// Convert a symlink entry into an Object holding its target
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
size: int64(len(target)),
modTime: item.ModTime(),
isLink: true,
link: target,
}
}
// readlink returns the target of the symlink the info describes.
//
// A symlink may carry an empty target, so the caller decides what is a
// symlink from the mode rather than from the target being set.
func readlink(info os.FileInfo) (string, error) {
st, ok := info.Sys().(*squashfs.StatT)
if !ok {
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
}
return st.LinkTarget, nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
defer recoverParsePanic(&err)
if f.file != "" && dir != "" {
return nil, fs.ErrorDirNotFound
}
nativeDir, err := f.toNative(dir)
if err != nil {
return nil, err
}
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
if err != nil {
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
}
entries = make(fs.DirEntries, 0, len(items))
skipped := 0
skippedLinks := 0
for _, item := range items {
// fs.Debugf(item.Name(), "entry = %#v", item)
// Skip entry names that aren't safe
if err := sanitize.Leaf(item.Name()); err != nil {
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
skipped++
continue
}
var entry fs.DirEntry
if item.IsDir() {
remote := f.fromNative(nativeDir, item.Name())
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = fs.NewDir(remote, info.ModTime())
} else if item.Type().IsRegular() {
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = f.objectFromFileInfo(nativeDir, info)
} else if item.Type()&os.ModeSymlink != 0 {
if !f.links {
skippedLinks++
continue
}
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
target, err := readlink(info)
if err != nil {
return nil, err
}
entry = f.objectFromLink(nativeDir, info, target)
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
}
if f.file != "" && entry.Remote() != f.file {
continue
}
entries = append(entries, entry)
}
// Log once per listing rather than per entry so a crafted image
// can't flood the log
if skipped > 0 {
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
}
if skippedLinks > 0 {
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
}
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
return entries, nil
}
// newObjectNative finds the object at the native path passed in
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
defer recoverParsePanic(&err)
// get the path and filename
dir, leaf := path.Split(nativePath)
dir = strings.TrimRight(dir, "/")
leaf = strings.Trim(leaf, "/")
// FIXME need to detect directory not found
fis, err := f.sqfs.ReadDir(toIOFS(dir))
if err != nil {
return nil, fs.ErrorObjectNotFound
}
for _, fi := range fis {
// Name the entry as List does, so that the two agree on
// which names exist
isLink := fi.Type()&os.ModeSymlink != 0
if isLink && !f.links {
continue
}
name := fi.Name()
if isLink {
name += fs.LinkSuffix
}
if name == leaf {
if fi.IsDir() {
return nil, fs.ErrorNotAFile
}
info, err := fi.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
}
if isLink {
target, err := readlink(info)
if err != nil {
return nil, err
}
o = f.objectFromLink(dir, info, target)
break
}
o = f.objectFromFileInfo(dir, info)
break
}
}
if o == nil {
return nil, fs.ErrorObjectNotFound
}
return o, nil
}
// NewObject finds the Object at remote.
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
if f.file != "" && remote != f.file {
return nil, fs.ErrorObjectNotFound
}
nativePath, err := f.toNative(remote)
if err != nil {
return nil, err
}
return f.newObjectNative(nativePath)
}
// Precision of the ModTimes in this Fs
func (f *Fs) Precision() time.Duration {
return time.Second
}
// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
return vfs.EROFS
}
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return vfs.EROFS
}
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
return nil, vfs.EROFS
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.None)
}
// UnWrap returns the Fs that this Fs is wrapping
func (f *Fs) UnWrap() fs.Fs {
return f.f
}
// WrapFs returns the Fs that is wrapping this Fs
func (f *Fs) WrapFs() fs.Fs {
return f.wrapper
}
// SetWrapper sets the Fs that is wrapping this Fs
func (f *Fs) SetWrapper(wrapper fs.Fs) {
f.wrapper = wrapper
}
// Object describes an object to be read from the raw squashfs file
type Object struct {
fs *Fs
remote string
size int64
modTime time.Time
isLink bool // whether this is a symlink
link string // target of the symlink
}
// Fs returns read only access to the Fs that this object is part of
func (o *Object) Fs() fs.Info {
return o.fs
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.Remote()
}
// Turn a squashfs path into a full path for the parent Fs
// func (o *Object) path(remote string) string {
// return path.Join(o.fs.prefix, remote)
// }
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// Size returns the size of the file
func (o *Object) Size() int64 {
return o.size
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
return o.modTime
}
// SetModTime sets the modification time of the local fs object
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
return vfs.EROFS
}
// Storable raturns a boolean indicating if this object is storable
func (o *Object) Storable() bool {
return true
}
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
defer recoverParsePanic(&err)
var offset, limit int64 = 0, -1
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
case *fs.RangeOption:
offset, limit = x.Decode(o.Size())
default:
if option.Mandatory() {
fs.Logf(o, "Unsupported mandatory option: %v", option)
}
}
}
if o.isLink {
// Clamp offset into range to avoid panic
if offset < 0 {
offset = 0
}
if offset > int64(len(o.link)) {
offset = int64(len(o.link))
}
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
}
remote, err := o.fs.toNative(o.remote)
if err != nil {
return nil, err
}
fs.Debugf(o, "Opening %q", remote)
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
if err != nil {
return nil, err
}
// discard data from start as necessary
if offset > 0 {
_, err = fh.Seek(offset, io.SeekStart)
if err != nil {
return nil, err
}
}
rc = recoveringReadCloser{fh}
// If limited then don't return everything
if limit >= 0 {
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
return readers.NewLimitedReadCloser(rc, limit), nil
}
return rc, nil
}
// Update in to the object with the modTime given of the given size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
return vfs.EROFS
}
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
return vfs.EROFS
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.Wrapper = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
)