Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

chunker: fix NewFs when root points to composite multi-chunk file without metadata

nielash8524afa9ce68
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 42f8aa16f..fe0a55afd 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -101,8 +101,10 @@ var (
//
// And still chunker's primary function is to chunk large files
// rather than serve as a generic metadata container.
-const maxMetadataSize = 1023
-const maxMetadataSizeWritten = 255
+const (
+ maxMetadataSize = 1023
+ maxMetadataSizeWritten = 255
+)
// Current/highest supported metadata format.
const metadataVersion = 2
@@ -317,11 +319,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
// i.e. `rpath` does not exist in the wrapped remote, but chunker
// detects a composite file because it finds the first chunk!
// (yet can't satisfy fstest.CheckListing, will ignore)
- if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
+ if err == nil && !f.useMeta {
firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
- _, testErr := cache.Get(ctx, baseName+firstChunkPath)
+ newBase, testErr := cache.Get(ctx, baseName+firstChunkPath)
if testErr == fs.ErrorIsFile {
+ f.base = newBase
err = testErr
+ cache.PinUntilFinalized(f.base, f)
}
}
@@ -972,7 +976,7 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
}
continue
}
- //fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
+ // fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
if err := o.addChunk(entry, chunkNo); err != nil {
return nil, err
}
@@ -1134,8 +1138,8 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
// put implements Put, PutStream, PutUnchecked, Update
func (f *Fs) put(
ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
- basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
-
+ basePut putFn, action string, target fs.Object,
+) (obj fs.Object, err error) {
// Perform consistency checks
if err := f.forbidChunk(src, remote); err != nil {
return nil, fmt.Errorf("%s refused: %w", action, err)
@@ -1956,7 +1960,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
return
}
wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
- //fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+ // fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
if entryType == fs.EntryObject {
mainPath, _, _, xactID := f.parseChunkName(path)
metaXactID := ""