Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

chunker: fix panic listing a directory after an interrupted upload with meta_format none

Nick Craig-Wood475997b6dec9
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 95d32d9be..45063c822 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -801,9 +801,12 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
fs.Debugf(f, "skip orphan data chunk %q", remote)
break
}
- if mainObject == nil && !f.useMeta {
+ if mainObject == nil && !f.useMeta && !isSpecial {
// this is the "nometa" case
// create dummy chunked object without metadata
+ //
+ // not for temporary or control chunks as they may be all
+ // that is left of an interrupted upload
mainObject = f.newObject(mainRemote, nil, nil)
byRemote[mainRemote] = mainObject
if !badEntry[mainRemote] {
diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.go
index 2f9f1fef0..6285f3b70 100644
--- a/backend/chunker/chunker_internal_test.go
+++ b/backend/chunker/chunker_internal_test.go
@@ -627,6 +627,22 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
_ = file.Remove(ctx)
}
+// testOrphanTempChunks checks that temporary chunks left behind by an
+// interrupted upload are not listed and don't break the listing
+func testOrphanTempChunks(t *testing.T, f *Fs) {
+ const dir = "orphan"
+ ctx := context.Background()
+ defer func() {
+ _ = operations.Purge(ctx, f.base, dir)
+ }()
+
+ modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+ item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
+ fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
+
+ fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+}
+
func testMetadataInput(t *testing.T, f *Fs) {
const minChunkForTest = 50
if f.opt.ChunkSize < minChunkForTest {
@@ -928,6 +944,9 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("BackwardsCompatibility", func(t *testing.T) {
testBackwardsCompatibility(t, f)
})
+ t.Run("OrphanTempChunks", func(t *testing.T) {
+ testOrphanTempChunks(t, f)
+ })
t.Run("ChunkerServerSideMove", func(t *testing.T) {
testChunkerServerSideMove(t, f)
})