Skip to content

Git designed for AI agents.

blobstore: publish file-bucket objects with link(2) instead of a flock

Federico Elia55f5abf8bf4a
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 37bbbc0..c80c76c 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -515,14 +515,23 @@ under a mutex created per writer (`fileblob.go` 764 and 789), which excludes
nothing, so two concurrent writers of one key both succeed and the second rename
overwrites the first; this holds within one process and between processes
sharing a file bucket. The phase-6 package doc's "fileblob supports IfNotExist" is
-true only for sequential writes and is corrected. For file buckets `Put` takes an
-exclusive `flock` on the key's parent directory (which the driver has created
-when it opened its temporary file) around the writer's `Close`: the stat and the
-rename of every `Put` into that directory are then serialized across processes on
-the host, so exactly one writer of a key wins. The lock adds no object (a
-directory is not listed) and the kernel releases it when a process dies. File
-buckets therefore need a local filesystem with `flock` (they are single-host
-already). S3, GCS and Azure use conditional requests, atomic on the server
+true only for sequential writes and is corrected. For file buckets `Put`
+publishes the object itself with **link(2)**, the filesystem's own atomic
+create-only primitive (PM review: an advisory lock binds only the writers that
+honour it, and lock semantics vary on network filesystems, while a file bucket
+shared by several hosts over NFS is a supported setup): fileblob's writer
+creates the temporary file `<path>.<hex nanos>.tmp` in the key's directory (so
+the key-to-path mapping stays fileblob's) and receives the content, `Put` fsyncs
+it, `os.Link`s it to the key's path (EEXIST is `ErrBlobExists`), discards the
+writer (whose cancelled `Close` only removes the temporary name) and fsyncs the
+directories. A network filesystem can report EEXIST for a link that succeeded
+but whose reply was lost and retransmitted; when the name is the same file as the
+temporary one (`os.SameFile`), the publish was this writer's. No `.attrs`
+sidecar is bypassed (file buckets are opened with `MetadataDontWrite`). File
+buckets therefore need a filesystem with hard links (every local Linux/macOS
+filesystem and NFS); one without them fails every `Put` with a clear error. The
+flock version of 9c8bb58 is replaced. The upstream defect is a stated limit in
+STORAGE.md. S3, GCS and Azure use conditional requests, atomic on the server
(untested here).
`List` uses `bucket.List(&blob.ListOptions{Prefix: prefix})` and skips `IsDir`
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 38fda4f..0501256 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -502,6 +502,16 @@ show:
- **S3, GCS and Azure are untested here.** The bucket is any gocloud URL, but
the test suite and the benchmark use `file://` and `mem://` buckets only;
the S3 tests run only when `GRAFT_TEST_S3_URL` points at a test bucket.
+- **File buckets need hard links.** The create-only write of gocloud's file
+ driver (v0.46.0, `IfNotExist`) is not atomic: its writer checks that the key
+ is absent and then renames its temporary file onto it, under a lock that
+ each writer creates for itself (`fileblob.go`, lines 764 and 789), so two
+ concurrent writers of one key both succeed and the second overwrites the
+ first. The storage core does not rely on it: it publishes each object of a
+ file bucket with link(2), which the filesystem makes atomic for every writer,
+ on one host or several sharing the directory over NFS. A file bucket therefore
+ needs a filesystem with hard links (every local Linux or macOS filesystem,
+ NFS); on one without them every write fails with an error that says so.
- **Concurrent writes to one repository share one lock** in authoritative
mode: reftable has one reference lock per repository. The voter holds it only
for the expected values and the database commit (the pack is uploaded before
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.go
index c397e62..53731f7 100644
--- a/internal/graft/corestore/blobstore/blobstore.go
+++ b/internal/graft/corestore/blobstore/blobstore.go
@@ -12,25 +12,32 @@
// gcsblob, azureblob, memblob and fileblob), so no plain-write fallback exists.
//
// Exactly one of concurrent Puts of a key wins. memblob checks and inserts
-// under its bucket-wide mutex. fileblob does not: its Close stats the key and
-// then renames the temporary file onto it, under a mutex created per writer, so
-// on its own two writers of one key would both succeed and the second rename
-// would overwrite the first, in one process or across processes sharing the
-// bucket. Put therefore holds an exclusive flock on the key's directory around
-// the writer's Close (lockDir); file buckets need a local filesystem with flock.
-// S3, GCS and Azure use conditional requests that the server decides.
+// under its bucket-wide mutex, and S3, GCS and Azure use conditional requests
+// that the server decides. fileblob does not: its writer's Close stats the key
+// and then renames the temporary file onto it, under a mutex created per writer
+// (gocloud.dev v0.46.0, fileblob.go 764 and 789), so two writers of one key both
+// succeed and the second rename overwrites the first, in one process or across
+// processes and hosts sharing the bucket. For file buckets Put therefore
+// publishes the object itself with link(2) (publishFile), which the filesystem
+// makes atomic and create-only for every writer: fileblob's writer creates the
+// temporary file (<path>.<hex nanos>.tmp, so the key-to-path mapping stays
+// fileblob's) and receives the content, the link gives it the key's name, and
+// the writer is then discarded, which only removes the temporary name. File
+// buckets therefore need a filesystem with hard links; one without them fails
+// every Put with a clear error. No .attrs sidecar is bypassed: file buckets are
+// opened with MetadataDontWrite.
//
// Atomic objects. gocloud writers commit the object on Close and discard it
// when the writer's context is cancelled before Close (probed: a cancelled
// writer leaves no object). Put cancels before Close on every failure, so a
// reader never sees a partial object.
//
-// Durability of file buckets. The fileblob driver renames its temporary file
-// into place on Close without any fsync. A changeset row in the database
-// (synchronous=FULL) can therefore outlive a pack whose data never reached the
-// disk. For file buckets Put fsyncs the content before Close, and fsyncs the
-// directories that hold the new entry (up to the bucket root) after Close. The
-// mem and remote drivers commit their objects durably themselves.
+// Durability of file buckets. Neither fileblob's writer nor a link fsyncs
+// anything, and a changeset row in the database (synchronous=FULL) could
+// outlive a pack whose data never reached the disk. For file buckets Put fsyncs
+// the content before the link publishes it, and fsyncs the directories that
+// hold the new entry (up to the bucket root) after it. The mem and remote
+// drivers commit their objects durably themselves.
//
// Listing. List reports objects only. fileblob also reports a writer's
// temporary file (<key>.<nanos>.tmp, in the key's directory because file
@@ -54,11 +61,12 @@ import (
"fmt"
"hash"
"io"
+ "io/fs"
"net/url"
"os"
"path/filepath"
+ "strconv"
"strings"
- "syscall"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"gocloud.dev/blob"
@@ -211,25 +219,30 @@ func (s *Store) Put(ctx context.Context, key string, r io.Reader) error {
}
if file != nil {
- // Close renames the file into place. The content must reach the disk
+ // The link below publishes the file. The content must reach the disk
// first: a database row may reference this object after a power loss.
if err := syncFile(file); err != nil {
return abort(fmt.Errorf("blobstore: sync %q: %w", key, err))
}
}
- // fileblob's Close stats the key and renames the temporary file onto it: a
- // create-only commit only while no other writer of a key in the directory does
- // the same, so it runs under the directory's lock (see the package doc). A
- // lock that cannot be taken aborts the write before Close publishes anything.
- unlock := func() {}
if file != nil {
- if unlock, err = lockDir(filepath.Dir(file.Name())); err != nil {
- return abort(err)
+ // fileblob's own Close is not a create-only publish (see the package doc):
+ // the content is published by a link, and the writer is then discarded
+ // (abort), which only removes the temporary name.
+ if err := publishFile(file.Name()); err != nil {
+ return abort(fmt.Errorf("blobstore: put %q: %w", key, err))
}
+ _ = abort(nil)
+
+ // The new name is durable only once its directory is synced.
+ if err := s.syncDirs(filepath.Dir(file.Name())); err != nil {
+ return fmt.Errorf("blobstore: sync directories of %q: %w", key, err)
+ }
+ return nil
}
+
closeErr := w.Close()
- unlock()
if gcerrors.Code(closeErr) == gcerrors.FailedPrecondition {
// The writer's Close discarded its content: the stored object is untouched.
return fmt.Errorf("blobstore: put %q: %w", key, corestore.ErrBlobExists)
@@ -237,15 +250,60 @@ func (s *Store) Put(ctx context.Context, key string, r io.Reader) error {
if closeErr != nil {
return fmt.Errorf("blobstore: close writer %q: %w", key, closeErr)
}
+ return nil
+}
- // The rename is durable only once its directory is synced.
- if file != nil {
- if err := s.syncDirs(filepath.Dir(file.Name())); err != nil {
- return fmt.Errorf("blobstore: sync directories of %q: %w", key, err)
- }
+// publishFile gives tmp, the temporary file of a fileblob writer, the name of
+// its key, create-only and atomically: link(2) fails with EEXIST when the name
+// exists, for every writer of the filesystem, other hosts of a network
+// filesystem included. A network filesystem can also report EEXIST for a link
+// that succeeded but whose reply was lost and retransmitted: when the name is
+// then the same file as tmp, the link was this writer's and the publish
+// succeeded. A conflict is corestore.ErrBlobExists.
+func publishFile(tmp string) error {
+ final, ok := finalPath(tmp)
+ if !ok {
+ return fmt.Errorf("unexpected temporary file name %q", tmp)
}
- return nil
+ err := os.Link(tmp, final)
+ if err == nil {
+ return nil
+ }
+ if !errors.Is(err, fs.ErrExist) {
+ return fmt.Errorf("link %q (file buckets need a filesystem with hard links): %w", final, err)
+ }
+
+ tmpInfo, statErr := os.Stat(tmp)
+ if statErr != nil {
+ return fmt.Errorf("stat %q after a link conflict: %w", tmp, statErr)
+ }
+ finalInfo, statErr := os.Stat(final)
+ if statErr != nil {
+ return fmt.Errorf("stat %q after a link conflict: %w", final, statErr)
+ }
+ if os.SameFile(tmpInfo, finalInfo) {
+ return nil
+ }
+ return corestore.ErrBlobExists
+}
+
+// finalPath returns the path of the object whose temporary file fileblob named
+// tmp: "<path>.<hex nanoseconds>.tmp" in the object's directory (file buckets
+// are opened with NoTempDir).
+func finalPath(tmp string) (string, bool) {
+ rest, ok := strings.CutSuffix(tmp, ".tmp")
+ if !ok {
+ return "", false
+ }
+ i := strings.LastIndex(rest, ".")
+ if i <= 0 || i == len(rest)-1 {
+ return "", false
+ }
+ if _, err := strconv.ParseUint(rest[i+1:], 16, 64); err != nil {
+ return "", false
+ }
+ return rest[:i], true
}
// syncFile fsyncs the temporary file of a file bucket writer.
@@ -259,8 +317,8 @@ func syncFile(f *os.File) error {
}
// syncDirs fsyncs dir and, when dir is inside the bucket root, each of its
-// ancestors up to the root: the directories that Put or the driver may have
-// created, and the entry that the rename created.
+// ancestors up to the root: the directories that the driver may have created,
+// and the entry that the link created.
func (s *Store) syncDirs(dir string) error {
inside := dir == s.root || strings.HasPrefix(dir, s.root+string(filepath.Separator))
for d := dir; ; d = filepath.Dir(d) {
@@ -323,26 +381,6 @@ func (s *Store) Exists(ctx context.Context, key string) (bool, error) {
return exists, nil
}
-// lockDir takes an exclusive flock on the directory dir and returns the
-// function that releases it. The lock lives on the open descriptor: closing it
-// releases the lock, and so does the death of the process.
-func lockDir(dir string) (unlock func(), _ error) {
- d, err := os.Open(dir)
- if err != nil {
- return nil, fmt.Errorf("blobstore: open directory to lock: %w", err)
- }
- if err := syscall.Flock(int(d.Fd()), syscall.LOCK_EX); err != nil {
- _ = d.Close()
- return nil, fmt.Errorf("blobstore: lock directory: %w", err)
- }
- return func() {
- // close(2) releases the descriptor, and with it the lock, even when it
- // reports an error, and a read-only directory descriptor has no pending
- // writes whose loss an error could signal: there is nothing to handle.
- _ = d.Close()
- }, nil
-}
-
// List implements corestore.BlobStore.
func (s *Store) List(ctx context.Context, prefix string, fn func(corestore.BlobInfo) error) error {
it := s.bucket.List(&blob.ListOptions{Prefix: prefix})
diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.go
index 2a685e6..b7b408a 100644
--- a/internal/graft/corestore/blobstore/durable_internal_test.go
+++ b/internal/graft/corestore/blobstore/durable_internal_test.go
@@ -121,6 +121,68 @@ func TestPutWithoutFileSyncHookSucceeds(t *testing.T) {
require.Equal(t, content, data)
}
+func TestPublishFileIsCreateOnly(t *testing.T) {
+ dir := t.TempDir()
+ write := func(name, content string) string {
+ path := filepath.Join(dir, name)
+ require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
+ return path
+ }
+ final := filepath.Join(dir, "k.pack")
+
+ first := write("k.pack.18f3a.tmp", "first")
+ require.NoError(t, publishFile(first))
+ // A network filesystem that lost the reply of a successful link retransmits
+ // it and reports EEXIST: the name is then the same file, and the publish
+ // succeeded.
+ require.NoError(t, publishFile(first))
+
+ second := write("k.pack.18f3b.tmp", "second")
+ require.ErrorIs(t, publishFile(second), corestore.ErrBlobExists)
+
+ data, err := os.ReadFile(final)
+ require.NoError(t, err)
+ require.Equal(t, "first", string(data), "the first publish is kept")
+}
+
+func TestFinalPath(t *testing.T) {
+ for tmp, want := range map[string]string{
+ "/b/repos/1/packs/k.pack.18f3a2b4c.tmp": "/b/repos/1/packs/k.pack",
+ "/b/plain.ff.tmp": "/b/plain",
+ } {
+ got, ok := finalPath(tmp)
+ require.True(t, ok, tmp)
+ require.Equal(t, want, got)
+ }
+ for _, tmp := range []string{"/b/k.pack", "/b/k.pack.tmp", "/b/k.pack.xyz.tmp", "/b/k.pack..tmp", ".1.tmp"} {
+ _, ok := finalPath(tmp)
+ require.False(t, ok, tmp)
+ }
+}
+
+func TestPutLeavesOnlyTheObject(t *testing.T) {
+ // The writer's temporary name is removed after the link published the
+ // object: only the object and its directories remain.
+ dir := t.TempDir()
+ s := openFileStoreForTest(t, dir, nil)
+
+ content := []byte("published by a link")
+ key := corestore.NewPackKey(4, sha256.Sum256(content))
+ require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))
+
+ var files []string
+ require.NoError(t, filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if !d.IsDir() {
+ files = append(files, path)
+ }
+ return nil
+ }))
+ require.Equal(t, []string{filepath.Join(dir, filepath.FromSlash(key))}, files)
+}
+
func TestPutMemBucketWithoutFileSync(t *testing.T) {
// mem buckets have no *os.File: Put must not require one and must not sync.
s, err := Open(testhelper.Context(t), "mem://")