Skip to content

Git designed for AI agents.

blobstore: strictly create-only Put, List and Delete (PLAN7 WP72b)

Federico Elia9c8bb58b5ba7
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.go
index 9a0b9c3..c397e62 100644
--- a/internal/graft/corestore/blobstore/blobstore.go
+++ b/internal/graft/corestore/blobstore/blobstore.go
@@ -1,18 +1,24 @@
// Package blobstore implements corestore.BlobStore on a gocloud bucket. Owned
// by WP52 (doc/graft/PLAN6.md).
//
-// Conditional writes. Put writes with blob.WriterOptions{IfNotExist: true}.
-// This was probed experimentally with gocloud.dev v0.46.0: the memblob and
-// fileblob drivers support IfNotExist, and an existing key is reported by
-// Close as gcerrors.FailedPrecondition. (The gocloud documentation names
-// gcerrors.PreconditionFailed, which these drivers do not return.) Put treats
-// FailedPrecondition as success, and it checks Exists first, so an existing
-// key never reads the content. No plain-write fallback is implemented for a
-// driver that reports gcerrors.Unimplemented: no registered driver does so
-// (checked against gocloud.dev v0.46.0: s3blob, gcsblob, azureblob, memblob and
-// fileblob all implement IfNotExist), and by the time Close reports it the
-// reader is consumed, so a fallback would need a second copy of the content.
-// Such an error is returned instead.
+// Create-only writes. Every upload uses a new owned key (corestore.NewPackKey),
+// so an object is never shared between two writers, and Put never succeeds on a
+// key that exists: it returns corestore.ErrBlobExists and leaves the stored
+// object as it is (doc/graft/PLAN7.md, PACK OWNERSHIP RULE). Put writes with
+// blob.WriterOptions{IfNotExist: true}; the drivers report an existing key from
+// the writer's Close as gcerrors.FailedPrecondition (the gocloud documentation
+// names gcerrors.PreconditionFailed, which they do not return). Every registered
+// driver implements IfNotExist (checked against gocloud.dev v0.46.0: s3blob,
+// gcsblob, azureblob, memblob and fileblob), so no plain-write fallback exists.
+//
+// Exactly one of concurrent Puts of a key wins. memblob checks and inserts
+// under its bucket-wide mutex. fileblob does not: its Close stats the key and
+// then renames the temporary file onto it, under a mutex created per writer, so
+// on its own two writers of one key would both succeed and the second rename
+// would overwrite the first, in one process or across processes sharing the
+// bucket. Put therefore holds an exclusive flock on the key's directory around
+// the writer's Close (lockDir); file buckets need a local filesystem with flock.
+// S3, GCS and Azure use conditional requests that the server decides.
//
// Atomic objects. gocloud writers commit the object on Close and discard it
// when the writer's context is cancelled before Close (probed: a cancelled
@@ -22,10 +28,15 @@
// Durability of file buckets. The fileblob driver renames its temporary file
// into place on Close without any fsync. A changeset row in the database
// (synchronous=FULL) can therefore outlive a pack whose data never reached the
-// disk, and because Put skips existing keys, such a pack would never be
-// repaired. For file buckets Put fsyncs the content before Close, and fsyncs
-// the directories that hold the new entry (up to the bucket root) after Close.
-// The mem and remote drivers commit their objects durably themselves.
+// disk. For file buckets Put fsyncs the content before Close, and fsyncs the
+// directories that hold the new entry (up to the bucket root) after Close. The
+// mem and remote drivers commit their objects durably themselves.
+//
+// Listing. List reports objects only. fileblob also reports a writer's
+// temporary file (<key>.<nanos>.tmp, in the key's directory because file
+// buckets are opened with NoTempDir) while the write is in flight, and one that
+// a crashed writer left behind. Callers that act on a listing (the garbage
+// collector) must treat keys they did not write accordingly.
//
// Integrity. Pack keys (corestore.ParsePackKey) are verified: Put hashes the
// content while writing and refuses content that does not match its key, and
@@ -47,6 +58,7 @@ import (
"os"
"path/filepath"
"strings"
+ "syscall"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"gocloud.dev/blob"
@@ -152,14 +164,6 @@ func withS3DefaultChecksumCalculation(parsed *url.URL) string {
// Put implements corestore.BlobStore.
func (s *Store) Put(ctx context.Context, key string, r io.Reader) error {
- exists, err := s.bucket.Exists(ctx, key)
- if err != nil {
- return fmt.Errorf("blobstore: check %q: %w", key, err)
- }
- if exists {
- return nil
- }
-
writeCtx, cancel := context.WithCancel(ctx)
defer cancel()
@@ -214,14 +218,27 @@ func (s *Store) Put(ctx context.Context, key string, r io.Reader) error {
}
}
+ // fileblob's Close stats the key and renames the temporary file onto it: a
+ // create-only commit only while no other writer of a key in the directory does
+ // the same, so it runs under the directory's lock (see the package doc). A
+ // lock that cannot be taken aborts the write before Close publishes anything.
+ unlock := func() {}
+ if file != nil {
+ if unlock, err = lockDir(filepath.Dir(file.Name())); err != nil {
+ return abort(err)
+ }
+ }
closeErr := w.Close()
- if closeErr != nil && gcerrors.Code(closeErr) != gcerrors.FailedPrecondition {
+ unlock()
+ if gcerrors.Code(closeErr) == gcerrors.FailedPrecondition {
+ // The writer's Close discarded its content: the stored object is untouched.
+ return fmt.Errorf("blobstore: put %q: %w", key, corestore.ErrBlobExists)
+ }
+ if closeErr != nil {
return fmt.Errorf("blobstore: close writer %q: %w", key, closeErr)
}
- // The rename (or, with FailedPrecondition, the key that another writer
- // created first, whose content is the same) is durable only once its
- // directory is synced.
+ // The rename is durable only once its directory is synced.
if file != nil {
if err := s.syncDirs(filepath.Dir(file.Name())); err != nil {
return fmt.Errorf("blobstore: sync directories of %q: %w", key, err)
@@ -306,6 +323,54 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) {
return exists, nil
}
+// lockDir takes an exclusive flock on the directory dir and returns the
+// function that releases it. The lock lives on the open descriptor: closing it
+// releases the lock, and so does the death of the process.
+func lockDir(dir string) (unlock func(), _ error) {
+ d, err := os.Open(dir)
+ if err != nil {
+ return nil, fmt.Errorf("blobstore: open directory to lock: %w", err)
+ }
+ if err := syscall.Flock(int(d.Fd()), syscall.LOCK_EX); err != nil {
+ _ = d.Close()
+ return nil, fmt.Errorf("blobstore: lock directory: %w", err)
+ }
+ return func() {
+ // close(2) releases the descriptor, and with it the lock, even when it
+ // reports an error, and a read-only directory descriptor has no pending
+ // writes whose loss an error could signal: there is nothing to handle.
+ _ = d.Close()
+ }, nil
+}
+
+// List implements corestore.BlobStore.
+func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {
+ it := s.bucket.List(&blob.ListOptions{Prefix: prefix})
+ for {
+ obj, err := it.Next(ctx)
+ if errors.Is(err, io.EOF) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("blobstore: list %q: %w", prefix, err)
+ }
+ if obj.IsDir {
+ continue
+ }
+ if err := fn(corestore.BlobInfo{Key: obj.Key, Size: obj.Size}); err != nil {
+ return err
+ }
+ }
+}
+
+// Delete implements corestore.BlobStore.
+func (s *Store) Delete(ctx context.Context, key string) error {
+ if err := s.bucket.Delete(ctx, key); err != nil && gcerrors.Code(err) != gcerrors.NotFound {
+ return fmt.Errorf("blobstore: delete %q: %w", key, err)
+ }
+ return nil
+}
+
// Close implements corestore.BlobStore.
func (s *Store) Close() error {
if err := s.bucket.Close(); err != nil {
diff --git a/internal/graft/corestore/blobstore/blobstore_test.go b/internal/graft/corestore/blobstore/blobstore_test.go
index 2436b48..a3cbec0 100644
--- a/internal/graft/corestore/blobstore/blobstore_test.go
+++ b/internal/graft/corestore/blobstore/blobstore_test.go
@@ -92,6 +92,10 @@ func TestPutGetRoundTrip(t *testing.T) {
}
}
+// TestPutIsCreateOnly checks the PLAN7 contract (PACK OWNERSHIP RULE 2): a Put
+// of an existing key fails with ErrBlobExists and leaves the stored object
+// unchanged. Before phase 7 it was a successful no-op; the requirement changed,
+// because a write must never rely on an object it did not write.
func TestPutIsCreateOnly(t *testing.T) {
for _, tb := range testBuckets(t) {
t.Run(tb.name, func(t *testing.T) {
@@ -99,41 +103,138 @@ func TestPutIsCreateOnly(t *testing.T) {
s := openStore(t, tb.url)
require.NoError(t, s.Put(ctx, "k/first", strings.NewReader("first")))
- // An existing key is success and its content is not read.
- require.NoError(t, s.Put(ctx, "k/first", failingReader{err: errors.New("must not be read")}))
+ require.ErrorIs(t, s.Put(ctx, "k/first", strings.NewReader("second")), corestore.ErrBlobExists)
data, err := readAll(t, s, "k/first")
require.NoError(t, err)
- require.Equal(t, "first", string(data))
+ require.Equal(t, "first", string(data), "the stored object is unchanged")
content := []byte("pack content, written twice")
pk := packKeyOf(content)
require.NoError(t, s.Put(ctx, pk, bytes.NewReader(content)))
- require.NoError(t, s.Put(ctx, pk, failingReader{err: errors.New("must not be read")}))
+ require.ErrorIs(t, s.Put(ctx, pk, bytes.NewReader(content)), corestore.ErrBlobExists)
+
+ data, err = readAll(t, s, pk)
+ require.NoError(t, err)
+ require.Equal(t, content, data)
})
}
}
+// TestPutConcurrentSameKey checks that exactly one of concurrent Puts of a key
+// wins (PLAN7, PACK OWNERSHIP RULE 2), and that the stored object is the
+// winner's: the others fail with ErrBlobExists. Before phase 7 every Put
+// succeeded; fileblob then let a later rename overwrite an earlier object.
func TestPutConcurrentSameKey(t *testing.T) {
+ const writers = 16
+
for _, tb := range testBuckets(t) {
t.Run(tb.name, func(t *testing.T) {
ctx := testhelper.Context(t)
s := openStore(t, tb.url)
- content := bytes.Repeat([]byte("0123456789abcdef"), 1<<14)
- pk := packKeyOf(content)
+ // A plain key, so that every writer can store different content and the
+ // stored object shows which writer won.
+ key := "race/plain"
+ contents := make([][]byte, writers)
+ for i := range contents {
+ contents[i] = bytes.Repeat([]byte{byte('a' + i)}, 1<<16)
+ }
+ errs := make([]error, writers)
+ start := make(chan struct{})
var g errgroup.Group
- for range 16 {
+ for i := range writers {
g.Go(func() error {
- return s.Put(ctx, pk, bytes.NewReader(content))
+ <-start
+ errs[i] = s.Put(ctx, key, bytes.NewReader(contents[i]))
+ return nil
})
}
+ close(start)
require.NoError(t, g.Wait())
- data, err := readAll(t, s, pk)
+ winner := -1
+ for i, err := range errs {
+ if err == nil {
+ require.Equal(t, -1, winner, "writers %d and %d both succeeded", winner, i)
+ winner = i
+ continue
+ }
+ require.ErrorIs(t, err, corestore.ErrBlobExists, "writer %d", i)
+ }
+ require.NotEqual(t, -1, winner, "no writer succeeded")
+
+ data, err := readAll(t, s, key)
require.NoError(t, err)
- require.Equal(t, content, data)
+ require.Equal(t, contents[winner], data, "the stored object is the winner's")
+ })
+ }
+}
+
+func TestList(t *testing.T) {
+ for _, tb := range testBuckets(t) {
+ t.Run(tb.name, func(t *testing.T) {
+ ctx := testhelper.Context(t)
+ s := openStore(t, tb.url)
+
+ want := map[string]int64{}
+ for _, key := range []string{"repos/1/packs/a.pack", "repos/1/packs/b.pack", "repos/1/other/c"} {
+ content := "content of " + key
+ require.NoError(t, s.Put(ctx, key, strings.NewReader(content)))
+ want[key] = int64(len(content))
+ }
+ require.NoError(t, s.Put(ctx, "repos/12/packs/d.pack", strings.NewReader("another repository")))
+
+ got := map[string]int64{}
+ require.NoError(t, s.List(ctx, "repos/1/", func(info corestore.BlobInfo) error {
+ got[info.Key] = info.Size
+ return nil
+ }))
+ require.Equal(t, want, got, "objects only, no directories, not repos/12")
+
+ stop := errors.New("stop")
+ calls := 0
+ err := s.List(ctx, "repos/1/", func(corestore.BlobInfo) error {
+ calls++
+ return stop
+ })
+ require.ErrorIs(t, err, stop)
+ require.Equal(t, 1, calls, "an error from fn stops the iteration")
+
+ require.NoError(t, s.List(ctx, "nothing/here/", func(corestore.BlobInfo) error {
+ return errors.New("no object expected")
+ }))
+ })
+ }
+}
+
+func TestDelete(t *testing.T) {
+ for _, tb := range testBuckets(t) {
+ t.Run(tb.name, func(t *testing.T) {
+ ctx := testhelper.Context(t)
+ s := openStore(t, tb.url)
+
+ content := []byte("a pack to delete")
+ pk := packKeyOf(content)
+ require.NoError(t, s.Put(ctx, pk, bytes.NewReader(content)))
+ require.NoError(t, s.Put(ctx, "keep/me", strings.NewReader("kept")))
+
+ require.NoError(t, s.Delete(ctx, pk))
+ exists, err := s.Exists(ctx, pk)
+ require.NoError(t, err)
+ require.False(t, exists)
+
+ // Deleting a missing key is success, so a collection can run again.
+ require.NoError(t, s.Delete(ctx, pk))
+ require.NoError(t, s.Delete(ctx, "never/written"))
+
+ data, err := readAll(t, s, "keep/me")
+ require.NoError(t, err)
+ require.Equal(t, "kept", string(data))
+
+ // The deleted key can be written again by a new Put.
+ require.NoError(t, s.Put(ctx, pk, bytes.NewReader(content)))
})
}
}
@@ -278,7 +379,7 @@ func TestS3Bucket(t *testing.T) {
content := []byte(t.Name() + time.Now().UTC().Format(time.RFC3339Nano))
pk := packKeyOf(content)
require.NoError(t, s.Put(ctx, pk, bytes.NewReader(content)))
- require.NoError(t, s.Put(ctx, pk, failingReader{err: errors.New("must not be read")}))
+ require.ErrorIs(t, s.Put(ctx, pk, bytes.NewReader(content)), corestore.ErrBlobExists)
data, err := readAll(t, s, pk)
require.NoError(t, err)
diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.go
index 3c567db..2a685e6 100644
--- a/internal/graft/corestore/blobstore/durable_internal_test.go
+++ b/internal/graft/corestore/blobstore/durable_internal_test.go
@@ -104,7 +104,9 @@ func TestPutFailedFileSyncLeavesNoObject(t *testing.T) {
func TestPutWithoutFileSyncHookSucceeds(t *testing.T) {
// The fsync steps run for real on a file bucket: the write is observable
- // afterwards and a repeated Put is still a no-op.
+ // afterwards. A second Put of the key is refused and leaves the object as it
+ // is (PLAN7, PACK OWNERSHIP RULE 2: Put is strictly create-only; before phase 7
+ // a repeated Put was a successful no-op).
dir := t.TempDir()
s := openFileStoreForTest(t, dir, nil)
ctx := testhelper.Context(t)
@@ -112,11 +114,11 @@ func TestPutWithoutFileSyncHookSucceeds(t *testing.T) {
content := []byte("durable content")
key := corestore.NewPackKey(2, sha256.Sum256(content))
require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))
- require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))
+ require.ErrorIs(t, s.Put(ctx, key, bytes.NewReader(content)), corestore.ErrBlobExists)
- exists, err := s.Exists(ctx, key)
+ data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(key)))
require.NoError(t, err)
- require.True(t, exists)
+ require.Equal(t, content, data)
}
func TestPutMemBucketWithoutFileSync(t *testing.T) {
diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.go
index c1957ab..7e36f21 100644
--- a/internal/graft/corestore/core.go
+++ b/internal/graft/corestore/core.go
@@ -10,12 +10,20 @@ import (
"github.com/feder-cr/git-ai/internal/gitaly/storage"
)
-// BlobStore is the object storage of the storage core. Keys are
-// content-addressed (see PackKey), so objects are immutable: Put never
-// overwrites. Implementations: blobstore (any gocloud bucket URL).
+// BlobStore is the object storage of the storage core. Objects are immutable
+// and every pack object belongs to exactly one database row: each upload uses a
+// new owned key (NewPackKey), and Put never overwrites nor shares an object
+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE). Implementations: blobstore (any
+// gocloud bucket URL).
type BlobStore interface {
- // Put stores the content under key if the key does not exist yet. An
- // existing key is success: its content is the same by construction.
+ // Put stores the content under key, which must not exist: an existing key is
+ // ErrBlobExists and the stored object is left unchanged. Of concurrent Puts
+ // of one key exactly one succeeds. For a pack key (see ParsePackKey) the
+ // content must hash to the key's SHA-256, else ErrBlobCorrupt. A Put whose
+ // context ends before it commits stores nothing. After any error the caller
+ // must not reference the key: an object a failed Put may have left (its
+ // commit succeeded but making it durable failed) is an orphan that garbage
+ // collection reclaims, never an object another writer relies on.
Put(ctx context.Context, key string, r io.Reader) error
// Get returns the content, or ErrBlobNotFound. For a pack key (see
// ParsePackKey) the reader returns ErrBlobCorrupt instead of io.EOF when
@@ -23,10 +31,25 @@ type BlobStore interface {
Get(ctx context.Context, key string) (io.ReadCloser, error)
// Exists reports whether the key exists.
Exists(ctx context.Context, key string) (bool, error)
+ // List calls fn for every object whose key starts with prefix, in no
+ // defined order. Directories are not objects. An error returned by fn stops
+ // the iteration and is returned as it is.
+ List(ctx context.Context, prefix string, fn func(BlobInfo) error) error
+ // Delete removes the object. A key that does not exist is success, so a
+ // collection that is run again never fails on an object it deleted before.
+ Delete(ctx context.Context, key string) error
// Close releases the bucket.
Close() error
}
+// BlobInfo describes a stored object. It has no modification time on purpose:
+// garbage collection never decides on bucket clocks (doc/graft/PLAN7.md, CLOCK
+// RULE).
+type BlobInfo struct {
+ Key string
+ Size int64
+}
+
// Watermark is how far the local cache of a repository is: every changeset
// of repository RepoID up to and including Seq is applied locally.
type Watermark struct {
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index af22f51..51fe9c8 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -182,6 +182,9 @@ var (
ErrRepositoryExists = errors.New("storage core: repository already exists")
// ErrBlobNotFound is returned by BlobStore.Get for a missing key.
ErrBlobNotFound = errors.New("storage core: blob not found")
+ // ErrBlobExists is returned by BlobStore.Put for a key that exists already:
+ // objects are never overwritten nor shared between writers.
+ ErrBlobExists = errors.New("storage core: blob already exists")
// ErrBlobCorrupt is returned by the reader of BlobStore.Get when the content
// does not hash to its key.
ErrBlobCorrupt = errors.New("storage core: blob content does not match its key")