Git designed for AI agents.
blobstore: list file buckets without failing on concurrent deletes
Federico Elia5c582fd1cd86
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.goindex 53731f7..a3f53e6 100644--- a/internal/graft/corestore/blobstore/blobstore.go+++ b/internal/graft/corestore/blobstore/blobstore.go@@ -381,8 +381,14 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) {return exists, nil}-// List implements corestore.BlobStore.+// List implements corestore.BlobStore. An object deleted while the listing+// runs is either listed or not, never an error: garbage collection deletes+// objects while other runs list them.func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {+ if s.root != "" {+ return s.listFiles(ctx, prefix, fn)+ }+it := s.bucket.List(&blob.ListOptions{Prefix: prefix})for {obj, err := it.Next(ctx)@@ -401,6 +407,66 @@ func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobI}}+// listFiles lists a file bucket by walking its directory. fileblob's own+// listing reads a directory and then stats each entry, and fails with NotFound+// when an entry was deleted in between; here such an entry is an object that is+// gone and is not listed. Keys are the slash-separated paths below the root+// (the storage core writes no key that fileblob would escape); directories are+// not objects.+func (s *Store) listFiles(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {+ start := s.root+ if i := strings.LastIndex(prefix, "/"); i >= 0 {+ start = filepath.Join(s.root, filepath.FromSlash(prefix[:i]))+ }++ // The error of fn is returned as it is (the BlobStore contract).+ var fnErr error+ err := filepath.WalkDir(start, func(path string, d fs.DirEntry, err error) error {+ if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ // The directory, or the entry, is gone: nothing to list there.+ return nil+ }+ return err+ }+ if err := ctx.Err(); err != nil {+ return err+ }+ if d.IsDir() {+ return nil+ }++ rel, err := filepath.Rel(s.root, path)+ if err != nil {+ return err+ }+ key := filepath.ToSlash(rel)+ if !strings.HasPrefix(key, prefix) {+ return nil+ }++ info, err := d.Info()+ if errors.Is(err, fs.ErrNotExist) {+ return nil+ }+ if err != nil {+ return err+ }+ if err := fn(corestore.BlobInfo{Key: key, Size: info.Size()}); err != nil {+ fnErr = err+ return fs.SkipAll+ }+ return nil+ })+ if fnErr != nil {+ return fnErr+ }+ if err != nil {+ return fmt.Errorf("blobstore: list %q: %w", prefix, err)+ }+ return nil+}+// Delete implements corestore.BlobStore.func (s *Store) Delete(ctx context.Context, key string) error {if err := s.bucket.Delete(ctx, key); err != nil && gcerrors.Code(err) != gcerrors.NotFound {diff --git a/internal/graft/corestore/blobstore/blobstore_test.go b/internal/graft/corestore/blobstore/blobstore_test.goindex a3cbec0..b98884b 100644--- a/internal/graft/corestore/blobstore/blobstore_test.go+++ b/internal/graft/corestore/blobstore/blobstore_test.go@@ -4,6 +4,7 @@ import ("bytes""crypto/sha256""errors"+ "fmt""io""os""path/filepath"@@ -209,6 +210,52 @@ func TestList(t *testing.T) {}}+// TestListWhileDeleting: an object deleted while a listing runs is listed or+// not, never an error. Garbage collection runs overlap: one deletes what+// another lists (fileblob's own listing failed with NotFound here).+func TestListWhileDeleting(t *testing.T) {+ for _, tb := range testBuckets(t) {+ t.Run(tb.name, func(t *testing.T) {+ ctx := testhelper.Context(t)+ s := openStore(t, tb.url)++ const objects = 200+ keys := make([]string, 0, objects)+ for i := range objects {+ key := fmt.Sprintf("repos/1/packs/%03d.pack", i)+ require.NoError(t, s.Put(ctx, key, strings.NewReader(key)))+ keys = append(keys, key)+ }++ deleted := make(chan struct{})+ var g errgroup.Group+ g.Go(func() error {+ defer close(deleted)+ for _, key := range keys {+ if err := s.Delete(ctx, key); err != nil {+ return err+ }+ }+ return nil+ })++ for listing := true; listing; {+ select {+ case <-deleted:+ listing = false+ default:+ }+ require.NoError(t, s.List(ctx, "repos/1/", func(corestore.BlobInfo) error { return nil }))+ }+ require.NoError(t, g.Wait())++ require.NoError(t, s.List(ctx, "repos/1/", func(info corestore.BlobInfo) error {+ return fmt.Errorf("object %q listed after its deletion", info.Key)+ }))+ })+ }+}+func TestDelete(t *testing.T) {for _, tb := range testBuckets(t) {t.Run(tb.name, func(t *testing.T) {diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.goindex 8920cf3..2f8cdee 100644--- a/internal/graft/corestore/core.go+++ b/internal/graft/corestore/core.go@@ -35,7 +35,8 @@ type BlobStore interface {Exists(ctx context.Context, key string) (bool, error)// List calls fn for every object whose key starts with prefix, in no// defined order. Directories are not objects. An error returned by fn stops- // the iteration and is returned as it is.+ // the iteration and is returned as it is. An object deleted while the+ // listing runs is listed or not, never an error (collections overlap).List(ctx context.Context, prefix string, fn func(BlobInfo) error) error// Delete removes the object. A key that does not exist is success, so a// collection that is run again never fails on an object it deleted before.