Skip to content

Git designed for AI agents.

blobstore: list file buckets without failing on concurrent deletes

Federico Elia5c582fd1cd86
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.go
index 53731f7..a3f53e6 100644
--- a/internal/graft/corestore/blobstore/blobstore.go
+++ b/internal/graft/corestore/blobstore/blobstore.go
@@ -381,8 +381,14 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) {
return exists, nil
}
-// List implements corestore.BlobStore.
+// List implements corestore.BlobStore. An object deleted while the listing
+// runs is either listed or not, never an error: garbage collection deletes
+// objects while other runs list them.
func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {
+ if s.root != "" {
+ return s.listFiles(ctx, prefix, fn)
+ }
+
it := s.bucket.List(&blob.ListOptions{Prefix: prefix})
for {
obj, err := it.Next(ctx)
@@ -401,6 +407,66 @@ func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobI
}
}
+// listFiles lists a file bucket by walking its directory. fileblob's own
+// listing reads a directory and then stats each entry, and fails with NotFound
+// when an entry was deleted in between; here such an entry is an object that is
+// gone and is not listed. Keys are the slash-separated paths below the root
+// (the storage core writes no key that fileblob would escape); directories are
+// not objects.
+func (s *Store) listFiles(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {
+ start := s.root
+ if i := strings.LastIndex(prefix, "/"); i >= 0 {
+ start = filepath.Join(s.root, filepath.FromSlash(prefix[:i]))
+ }
+
+ // The error of fn is returned as it is (the BlobStore contract).
+ var fnErr error
+ err := filepath.WalkDir(start, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ // The directory, or the entry, is gone: nothing to list there.
+ return nil
+ }
+ return err
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+
+ rel, err := filepath.Rel(s.root, path)
+ if err != nil {
+ return err
+ }
+ key := filepath.ToSlash(rel)
+ if !strings.HasPrefix(key, prefix) {
+ return nil
+ }
+
+ info, err := d.Info()
+ if errors.Is(err, fs.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if err := fn(corestore.BlobInfo{Key: key, Size: info.Size()}); err != nil {
+ fnErr = err
+ return fs.SkipAll
+ }
+ return nil
+ })
+ if fnErr != nil {
+ return fnErr
+ }
+ if err != nil {
+ return fmt.Errorf("blobstore: list %q: %w", prefix, err)
+ }
+ return nil
+}
+
// Delete implements corestore.BlobStore.
func (s *Store) Delete(ctx context.Context, key string) error {
if err := s.bucket.Delete(ctx, key); err != nil && gcerrors.Code(err) != gcerrors.NotFound {
diff --git a/internal/graft/corestore/blobstore/blobstore_test.go b/internal/graft/corestore/blobstore/blobstore_test.go
index a3cbec0..b98884b 100644
--- a/internal/graft/corestore/blobstore/blobstore_test.go
+++ b/internal/graft/corestore/blobstore/blobstore_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/sha256"
"errors"
+ "fmt"
"io"
"os"
"path/filepath"
@@ -209,6 +210,52 @@ func TestList(t *testing.T) {
}
}
+// TestListWhileDeleting: an object deleted while a listing runs is listed or
+// not, never an error. Garbage collection runs overlap: one deletes what
+// another lists (fileblob's own listing failed with NotFound here).
+func TestListWhileDeleting(t *testing.T) {
+ for _, tb := range testBuckets(t) {
+ t.Run(tb.name, func(t *testing.T) {
+ ctx := testhelper.Context(t)
+ s := openStore(t, tb.url)
+
+ const objects = 200
+ keys := make([]string, 0, objects)
+ for i := range objects {
+ key := fmt.Sprintf("repos/1/packs/%03d.pack", i)
+ require.NoError(t, s.Put(ctx, key, strings.NewReader(key)))
+ keys = append(keys, key)
+ }
+
+ deleted := make(chan struct{})
+ var g errgroup.Group
+ g.Go(func() error {
+ defer close(deleted)
+ for _, key := range keys {
+ if err := s.Delete(ctx, key); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+
+ for listing := true; listing; {
+ select {
+ case <-deleted:
+ listing = false
+ default:
+ }
+ require.NoError(t, s.List(ctx, "repos/1/", func(corestore.BlobInfo) error { return nil }))
+ }
+ require.NoError(t, g.Wait())
+
+ require.NoError(t, s.List(ctx, "repos/1/", func(info corestore.BlobInfo) error {
+ return fmt.Errorf("object %q listed after its deletion", info.Key)
+ }))
+ })
+ }
+}
+
func TestDelete(t *testing.T) {
for _, tb := range testBuckets(t) {
t.Run(tb.name, func(t *testing.T) {
diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.go
index 8920cf3..2f8cdee 100644
--- a/internal/graft/corestore/core.go
+++ b/internal/graft/corestore/core.go
@@ -35,7 +35,8 @@ type BlobStore interface {
Exists(ctx context.Context, key string) (bool, error)
// List calls fn for every object whose key starts with prefix, in no
// defined order. Directories are not objects. An error returned by fn stops
- // the iteration and is returned as it is.
+ // the iteration and is returned as it is. An object deleted while the
+ // listing runs is listed or not, never an error (collections overlap).
List(ctx context.Context, prefix string, fn func(BlobInfo) error) error
// Delete removes the object. A key that does not exist is success, so a
// collection that is run again never fails on an object it deleted before.