Git designed for AI agents.
quarantine: take a directory's descriptor only after a link into it
Federico Eliab5472f95adae
diff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.mdindex cb689b0..a43c4e0 100644--- a/doc/graft/LIMITATIONS.md+++ b/doc/graft/LIMITATIONS.md@@ -149,9 +149,12 @@ Git hooks use private.written just before a crash may be missing although the write succeeded.- Custom hooks get the server's `core.fsync` configuration, so the objects thattheir git commands write into a quarantine are durable. When a custom hook- runs with a quarantine, the migration of that quarantine also fsyncs its files,+ runs with a quarantine of the server's own write RPCs (land, commit, merge and+ the other operations), the migration of that quarantine also fsyncs its files,because a hook may write objects without git; that costs about one fsync per- file of the write.+ file of the write. Pushes over SSH or HTTP are different: git's receive-pack+ migrates its own quarantine, so objects that a custom hook writes there+ without git are only as durable as the hook makes them.### Landing (`UserLand`)diff --git a/internal/git/quarantine/migrate_test.go b/internal/git/quarantine/migrate_test.goindex 77d7e00..3cd7258 100644--- a/internal/git/quarantine/migrate_test.go+++ b/internal/git/quarantine/migrate_test.go@@ -101,6 +101,52 @@ func TestMigrateRecreatesTargetDirectoriesThatVanish(t *testing.T) {require.NoDirExists(t, source)}+// TestMigrateSyncsTheDirectoryThatHoldsTheLink removes each target fan-out directory right before+// the first link into it and creates it again, as housekeeping and then another writer may. The+// link then succeeds into the new directory. The second pass must fsync that directory, not the+// removed one: every descriptor it fsyncs is of the directory now at its path.+func TestMigrateSyncsTheDirectoryThatHoldsTheLink(t *testing.T) {+ ctx := testhelper.Context(t)+ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ fanouts := []string{"0a", "1b", "2c"}+ files := writeQuarantine(t, source, fanouts)+ require.NoError(t, os.MkdirAll(target, 0o755))++ var recreated sync.Map+ replaceHooks(t, func(targetPath string) {+ dir := filepath.Dir(targetPath)+ if filepath.Base(dir) == "pack" {+ return+ }+ if _, done := recreated.LoadOrStore(dir, true); !done {+ require.NoError(t, os.Remove(dir))+ require.NoError(t, os.Mkdir(dir, 0o755))+ }+ })++ var synced []string+ originalSyncDir := syncDirFile+ syncDirFile = func(f *os.File) error {+ descriptor, err := f.Stat()+ require.NoError(t, err)+ current, err := os.Stat(f.Name())+ require.NoError(t, err)+ require.True(t, os.SameFile(descriptor, current), "the second pass fsyncs a removed directory: %s", f.Name())+ synced = append(synced, f.Name())+ return originalSyncDir(f)+ }++ require.NoError(t, migrate(ctx, source, target, false))++ for _, file := range files {+ require.FileExists(t, filepath.Join(target, file))+ }+ for _, fanout := range fanouts {+ require.Contains(t, synced, filepath.Join(target, fanout))+ }+}+// TestMigrateSyncsEachDirectoryOnce counts the fsyncs: one per target directory that received a// link, one for the object directory, and none per file, whose writers made them durable.func TestMigrateSyncsEachDirectoryOnce(t *testing.T) {diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.goindex a04f67d..4e632d1 100644--- a/internal/git/quarantine/quarantine.go+++ b/internal/git/quarantine/quarantine.go@@ -168,9 +168,17 @@ func (d *Dir) Migrate(ctx context.Context) error {// empty and stays. Each round needs another removal between the creation and the link, so the loop// ends; it also ends with the context.//+// The descriptor of a target directory is taken right after a link into it succeeded, never+// before: the link pins the directory, so the descriptor is of the directory that holds the link.+// A descriptor taken earlier could be of a directory that housekeeping removes, and that another+// writer then creates again, before the link (the link then succeeds into the new directory, which+// the second pass would not fsync). If the directory is gone even after the link, housekeeping+// removed the linked object because a pack holds it (git prune-packed), and nothing of the+// migration is left there to make durable.+//// The second pass, unless a transaction manager owns durability (storage.NeedsSync), fsyncs each-// target directory that received a link, once, through a descriptor opened in the first pass, and-// the object directory itself. The files are not fsynced: they are durable before the migration+// target directory that received a link, once, through those descriptors, and the object+// directory itself. The files are not fsynced: they are durable before the migration// (see Dir), unless untrustedWriter is set, in which case they are fsynced in parallel before the// first pass.//@@ -192,9 +200,17 @@ func migrate(ctx context.Context, sourcePath, targetPath string, untrustedWriterm := migration{ctx: ctx, dirs: map[string]*os.File{}}defer m.closeDirs()- if err := m.openDir(targetPath); err != nil {+ // The object directory is not a fan-out directory, so housekeeping never removes it. Its+ // descriptor is taken first, so the second pass fsyncs it last.+ if err := ensureDir(targetPath); err != nil {+ return err+ }+ if err := m.track(targetPath); err != nil {return err}+ if _, ok := m.dirs[targetPath]; !ok {+ return fmt.Errorf("opening target directory %q: %w", targetPath, fs.ErrNotExist)+ }if err := m.link(sourcePath, targetPath); err != nil {return err}@@ -223,19 +239,45 @@ type migration struct {order []string}-// openDir creates the target directory dir if it does not exist and opens a descriptor of it,-// replacing the descriptor of a directory of the same path that was removed.-func (m *migration) openDir(dir string) error {+// ensureDir creates the target directory dir. An existing entry at dir is accepted as it is, as+// before: a file there (a conflict with the quarantine's directory) is left to the links into it+// to report.+func ensureDir(dir string) error {if err := os.Mkdir(dir, mode.Directory); err != nil && !errors.Is(err, os.ErrExist) {return fmt.Errorf("creating target directory %q: %w", dir, err)}+ return nil+}++// track keeps a descriptor of the target directory dir for the second pass. It runs after a link+// into dir succeeded (see migrate). It keeps the descriptor it has when that is still of the+// directory at dir, and replaces it otherwise. A dir that no longer exists holds nothing of the+// migration (housekeeping removed the linked object because a pack holds it), so it is skipped.+func (m *migration) track(dir string) error {+ info, err := os.Stat(dir)+ if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ return nil+ }+ return fmt.Errorf("checking target directory %q: %w", dir, err)+ }++ old, known := m.dirs[dir]+ if known {+ if oldInfo, err := old.Stat(); err == nil && os.SameFile(oldInfo, info) {+ return nil+ }+ }f, err := os.Open(dir)if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ return nil+ }return fmt.Errorf("opening target directory %q: %w", dir, err)}- if old, ok := m.dirs[dir]; ok {+ if known {_ = old.Close()} else {m.order = append(m.order, dir)@@ -264,7 +306,10 @@ func (m *migration) link(sourcePath, targetPath string) error {nestedSourcePath := filepath.Join(sourcePath, entry.Name())if entry.IsDir() {- if err := m.openDir(nestedTargetPath); err != nil {+ // The directory structure is migrated as it is, empty directories included. Its+ // descriptor is taken only after a link into it (linkFile), which also creates it+ // again if housekeeping removed it in between.+ if err := ensureDir(nestedTargetPath); err != nil {return err}if err := m.link(nestedSourcePath, nestedTargetPath); err != nil {@@ -288,12 +333,17 @@ func (m *migration) link(sourcePath, targetPath string) error {// linkFile links the file at sourcePath to targetPath, creating the target directory again when// it was removed (see migrate).func (m *migration) linkFile(sourcePath, targetPath string) error {+ dir := filepath.Dir(targetPath)for {+ if err := ensureDir(dir); err != nil {+ return err+ }+beforeLink(targetPath)err := finalizeObjectFile(sourcePath, targetPath)if err == nil {- return nil+ return m.track(dir)}if !errors.Is(err, fs.ErrNotExist) {return err@@ -306,10 +356,7 @@ func (m *migration) linkFile(sourcePath, targetPath string) error {return fmt.Errorf("%w (%w)", err, ctxErr)}- // The source exists, so the target directory is gone.- if err := m.openDir(filepath.Dir(targetPath)); err != nil {- return err- }+ // The source exists, so the target directory is gone: create it again and link again.}}