Git designed for AI agents.
land: a publish mode per caller; unbatched lands publish loose objects
Federico Eliaf653afae9e64
diff --git a/internal/git/localrepo/merge_tree_stream.go b/internal/git/localrepo/merge_tree_stream.goindex 34e5c04..984c84b 100644--- a/internal/git/localrepo/merge_tree_stream.go+++ b/internal/git/localrepo/merge_tree_stream.go@@ -368,9 +368,10 @@ type helperConfig struct {// WithoutObjectFsync starts the helper processes with core.fsync=none. A MergeTreeStream and a// CommitWriter write nothing but objects, so the option only drops the fsync of the objects they-// write. The caller must make those objects durable before any reference points to them: the batch-// stepper of UserLand packs every loose object of its quarantine into one fsynced pack, and the-// quarantine migration fsyncs whatever else it moves (see doc/graft/PLAN8.md, section 3.2).+// write. The caller must make those objects durable itself, before a quarantine that holds them is+// migrated: every object file in a quarantine must be durable at migration time, and the migration+// is not relied on to fsync them. The batch attempts of UserLand pack every loose object of their+// quarantine into one fsynced pack before the update (see doc/graft/PLAN8.md, section 3.2).//// The value replaces the core.fsync of the server for the helper. It cannot be derived from it: git// does not subtract a component from a value set earlier (a later "-loose-object" replaces thediff --git a/internal/gitaly/service/operations/land.go b/internal/gitaly/service/operations/land.goindex 4e61087..1641f40 100644--- a/internal/gitaly/service/operations/land.go+++ b/internal/gitaly/service/operations/land.go@@ -168,7 +168,7 @@ func (s *Server) landAncestry(ctx context.Context, quarantineRepo *localrepo.Rep// landAttempt performs a single compare-and-swap attempt of UserLand. It returns a landConflict// if the branch was moved by a concurrent writer, in which case the caller retries.func (s *Server) landAttempt(ctx context.Context, in *gitalypb.UserLandRequest, committer git.Signature) (*gitalypb.UserLandResponse, error) {- attempt, cleanup, err := s.newLandingAttempt(ctx, in.GetRepository(), ignoreLandPhase)+ attempt, cleanup, err := s.newLandingAttempt(ctx, in.GetRepository(), landPublishLoose, ignoreLandPhase)if err != nil {return nil, err}diff --git a/internal/gitaly/service/operations/land_attempt.go b/internal/gitaly/service/operations/land_attempt.goindex 7b637e2..ff0a179 100644--- a/internal/gitaly/service/operations/land_attempt.go+++ b/internal/gitaly/service/operations/land_attempt.go@@ -25,11 +25,27 @@ type landPhaseObserver func(phase string, start time.Time)// ignoreLandPhase is the landPhaseObserver of an unbatched land.func ignoreLandPhase(string, time.Time) {}+// landPublishMode is how a landing attempt makes the objects it writes durable. Both modes keep the+// durability contract of a quarantine: every object file in it is durable before the quarantine+// migration moves it into the repository, which therefore only has to make the links durable.+type landPublishMode int++const (+ // landPublishPack is the mode of a batch attempt. Its rebaser writes the objects of all members+ // without fsync, and publish packs every loose object of the quarantine into one pack, which git+ // pack-objects fsyncs, before the update. A pack is the cheap way to make many objects durable.+ landPublishPack landPublishMode = iota + 1+ // landPublishLoose is the mode of an unbatched attempt. Its rebaser writes the few objects of+ // one change with the server's fsync configuration, and the migration moves them as they are.+ // A pack would cost a process and as many fsyncs for these few objects, and the attempts that+ // lose the compare-and-swap would each leave a pack in the repository.+ landPublishLoose+)+// landingAttempt is one attempt to land changes onto a branch, in a quarantine of its own. The-// unbatched land and every attempt of a batch use it, so both decide how a change lands, rebase it-// and publish its objects the same way (doc/graft/PLAN8.md): one listing of the commits a change-// adds to the tip decides, one rebase engine replays, and the objects the attempt wrote are made-// durable once, as one pack, before the reference moves.+// unbatched land and every attempt of a batch use it, so both decide how a change lands and+// rebase it the same way (doc/graft/PLAN8.md): one listing of the commits a change adds to the tip+// decides, and one rebase engine replays. How the objects become durable is the mode of the attempt.type landingAttempt struct {server *Serverrepository *gitalypb.Repository@@ -45,6 +61,8 @@ type landingAttempt struct {ancestry ancestry.Checker// observe observes the phases of the attempt.observe landPhaseObserver+ // mode is how the attempt makes its objects durable.+ mode landPublishMode}// batchRebaser replays the commits of the changes of an attempt. It is rebase.Engine.@@ -53,17 +71,24 @@ type batchRebaser interface {Close() error}-// newBatchRebaser starts the rebaser of an attempt on the quarantine of the attempt. The objects it-// writes are not fsynced one by one: the publish of the attempt packs them into one fsynced pack-// before the reference points to them (see publish). It is a variable so that tests can observe the-// replays.-var newBatchRebaser = func(repo *localrepo.Repo) batchRebaser {- return rebase.New(repo, rebase.WithDeferredDurability())+// newBatchRebaser starts the rebaser of an attempt in mode on the quarantine of the attempt. Only+// the pack mode defers the durability of the objects it writes to the publish of the attempt (see+// landPublishPack); the loose mode writes them with the server's fsync configuration. It is a+// variable so that tests can observe the replays.+var newBatchRebaser = func(repo *localrepo.Repo, mode landPublishMode) batchRebaser {+ if mode == landPublishPack {+ return rebase.New(repo, rebase.WithDeferredDurability())+ }+ return rebase.New(repo)}-// newLandingAttempt starts an attempt on repository: it opens the quarantine of the attempt and the-// processes that serve it. The returned function ends the attempt.-func (s *Server) newLandingAttempt(ctx context.Context, repository *gitalypb.Repository, observe landPhaseObserver) (*landingAttempt, func(), error) {+// newLandingAttempt starts an attempt on repository in mode: it opens the quarantine of the attempt+// and the processes that serve it. The returned function ends the attempt.+func (s *Server) newLandingAttempt(ctx context.Context, repository *gitalypb.Repository, mode landPublishMode, observe landPhaseObserver) (*landingAttempt, func(), error) {+ if mode != landPublishPack && mode != landPublishLoose {+ return nil, nil, structerr.NewInternal("unknown publish mode %d", mode)+ }+defer observe(landBatchPhaseQuarantine, time.Now())quarantineDir, quarantineRepo, cleanup, err := s.quarantinedRepo(ctx, repository)@@ -92,9 +117,8 @@ func (s *Server) newLandingAttempt(ctx context.Context, repository *gitalypb.Repreturn nil, nil, structerr.NewInternal("reading object info: %w", err)}- // One rebase engine serves the whole attempt, and the attempt ends with it. Its objects are- // made durable by the publish of the attempt, as one pack (see publish).- rebaser := newBatchRebaser(quarantineRepo)+ // One rebase engine serves the whole attempt, and the attempt ends with it.+ rebaser := newBatchRebaser(quarantineRepo, mode)return &landingAttempt{server: s,@@ -106,6 +130,7 @@ func (s *Server) newLandingAttempt(ctx context.Context, repository *gitalypb.Reprebaser: rebaser,ancestry: checker,observe: observe,+ mode: mode,}, func() {releaseInfo()releaseChecker()@@ -328,11 +353,16 @@ func (a *landingAttempt) computeTip(ctx context.Context, in *gitalypb.UserLandRe}}-// publish makes the objects that the attempt wrote durable before its compare-and-swap: it packs-// every loose object of the quarantine into one fsynced pack (see packAttemptObjects), which the-// quarantine migration of the update then moves into the repository. A failure fails the update of-// the attempt: nothing is published.+// publish makes the objects that the attempt wrote durable before its compare-and-swap, so that the+// quarantine migration of the update only moves durable files. In the pack mode it packs every+// loose object of the quarantine into one fsynced pack (see packAttemptObjects); a failure fails the+// update of the attempt, and nothing is published. In the loose mode the objects were fsynced when+// they were written, and there is nothing to do.func (a *landingAttempt) publish(ctx context.Context) error {+ if a.mode == landPublishLoose {+ return nil+ }+defer a.observe(landBatchPhasePack, time.Now())repoPath, err := a.repo.Path(ctx)diff --git a/internal/gitaly/service/operations/land_attempt_test.go b/internal/gitaly/service/operations/land_attempt_test.goindex 99df632..ae5f219 100644--- a/internal/gitaly/service/operations/land_attempt_test.go+++ b/internal/gitaly/service/operations/land_attempt_test.go@@ -3,7 +3,11 @@ package operationsimport ("context""errors"+ "fmt"+ "os""path/filepath"+ "strings"+ "sync""sync/atomic""testing"@@ -12,79 +16,198 @@ import ("github.com/feder-cr/git-ai/internal/git/gittest""github.com/feder-cr/git-ai/internal/git/localrepo""github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/feder-cr/git-ai/proto/go/gitalypb""github.com/stretchr/testify/require"- "google.golang.org/grpc/codes")-// unbatchedLandFixture is a repository whose main branch is at base, with a change based on the+// unbatchedLandFixture is a repository whose main branch is at base, with changes based on the// parent of base, served by an operations service without batching (not standalone).-func unbatchedLandFixture(t *testing.T, ctx context.Context) (context.Context, stepperFixture, func(git.ObjectID) error) {+func unbatchedLandFixture(t *testing.T, ctx context.Context) (context.Context, stepperFixture, gitalypb.OperationServiceClient) {t.Helper()ctx, cfg, client := setupOperationsService(t, ctx)- f := newStepperFixture(t, ctx, cfg)+ return ctx, newStepperFixture(t, ctx, cfg), client+}- land := func(commit git.ObjectID) error {- resp, err := client.UserLand(ctx, landRequest(f.repoProto, git.DefaultBranch, commit))- if err != nil {- return err- }+// landUnbatched lands commit on the main branch of f and requires that it did not share a batch.+func landUnbatched(t *testing.T, ctx context.Context, client gitalypb.OperationServiceClient, f stepperFixture, commit git.ObjectID) (*gitalypb.UserLandResponse, error) {+ t.Helper()++ resp, err := client.UserLand(ctx, landRequest(f.repoProto, git.DefaultBranch, commit))+ if err == nil {require.Equal(t, int32(1), resp.GetBatchSize(), "the land is not batched")- return nil}-- return ctx, f, land+ return resp, err}-// TestUserLand_unbatchedLandUsesTheLandingAttempt is not parallel: it replaces newBatchRebaser. An-// unbatched land decides and rebases like a batch attempt (one replay through the rebaser of the-// attempt) and publishes its objects as one pack: the rebased commit is not a loose object of the-// repository, and the repository is connected.-func TestUserLand_unbatchedLandUsesTheLandingAttempt(t *testing.T) {+// TestUserLand_unbatchedLandPublishesLooseObjects is not parallel: it replaces newBatchRebaser and+// packBatchObjects. An unbatched land decides and rebases like a batch attempt (one replay through+// the rebaser of the attempt), in the loose publish mode: its rebaser writes with the server's fsync+// configuration, nothing is packed, and the migration moves its loose objects into the repository.+func TestUserLand_unbatchedLandPublishesLooseObjects(t *testing.T) {ctx := featureflag.OutgoingCtxWithFeatureFlag(testhelper.Context(t), featureflag.GPGSigning, false)var replays atomic.Int64- original := newBatchRebaser- newBatchRebaser = func(repo *localrepo.Repo) batchRebaser {- return countingRebaser{batchRebaser: original(repo), rebases: &replays}+ var modes sync.Map+ originalRebaser := newBatchRebaser+ newBatchRebaser = func(repo *localrepo.Repo, mode landPublishMode) batchRebaser {+ modes.Store(mode, true)+ return countingRebaser{batchRebaser: originalRebaser(repo, mode), rebases: &replays}+ }+ t.Cleanup(func() { newBatchRebaser = originalRebaser })++ originalPack := packBatchObjects+ packBatchObjects = func(context.Context, *localrepo.Repo, string) (int, error) {+ return 0, errors.New("an unbatched land must not pack")}- t.Cleanup(func() { newBatchRebaser = original })+ t.Cleanup(func() { packBatchObjects = originalPack })- _, f, land := unbatchedLandFixture(t, ctx)+ ctx, f, client := unbatchedLandFixture(t, ctx)packsBefore := len(packIndexes(t, filepath.Join(f.repoPath, "objects")))- require.NoError(t, land(f.diverged1))+ _, err := landUnbatched(t, ctx, client, f, f.diverged1)+ require.NoError(t, err)require.Equal(t, int64(1), replays.Load(), "the diverged change is replayed once, through the rebaser")+ _, loose := modes.Load(landPublishLoose)+ _, pack := modes.Load(landPublishPack)+ require.True(t, loose, "the rebaser of an unbatched land is in the loose mode")+ require.False(t, pack, "an unbatched land never uses the pack mode")+tip := gittest.ResolveRevision(t, f.cfg, f.repoPath, "refs/heads/"+git.DefaultBranch)require.NotEqual(t, f.diverged1, tip, "the change was rebased")- require.NoFileExists(t, filepath.Join(f.repoPath, "objects", tip.String()[:2], tip.String()[2:]),- "the rebased commit is not a loose object of the repository")- require.Len(t, packIndexes(t, filepath.Join(f.repoPath, "objects")), packsBefore+1, "the land publishes one pack")+ require.FileExists(t, filepath.Join(f.repoPath, "objects", tip.String()[:2], tip.String()[2:]),+ "the rebased commit is a loose object of the repository")+ require.Len(t, packIndexes(t, filepath.Join(f.repoPath, "objects")), packsBefore, "an unbatched land publishes no pack")gittest.Exec(t, f.cfg, "-C", f.repoPath, "fsck", "--connectivity-only", "--no-dangling")-- // A fast-forward writes no object, so it replays nothing and publishes no pack.- forward := gittest.WriteCommit(t, f.cfg, f.repoPath, gittest.WithParents(tip), gittest.WithMessage("forward"))- require.NoError(t, land(forward))- require.Equal(t, int64(1), replays.Load())- require.Len(t, packIndexes(t, filepath.Join(f.repoPath, "objects")), packsBefore+1)- requireBranchAt(t, f.cfg, f.repoPath, git.DefaultBranch, forward)}-// TestUserLand_unbatchedPackFailureFailsTheLand is not parallel: it replaces packBatchObjects. A land-// whose objects cannot be packed fails with Internal and leaves the branch where it was.-func TestUserLand_unbatchedPackFailureFailsTheLand(t *testing.T) {+// TestUserLand_contendedUnbatchedLandsLeaveNoPacks: unbatched lands that race on one branch lose+// compare-and-swaps and retry, and no attempt, lost or won, leaves a pack in the repository.+func TestUserLand_contendedUnbatchedLandsLeaveNoPacks(t *testing.T) {+ t.Parallel()+ctx := featureflag.OutgoingCtxWithFeatureFlag(testhelper.Context(t), featureflag.GPGSigning, false)+ ctx, f, client := unbatchedLandFixture(t, ctx)+ packsBefore := len(packIndexes(t, filepath.Join(f.repoPath, "objects")))++ rootEntry := gittest.TreeEntry{Mode: "100644", Path: "root", Content: "root"}+ root := gittest.ResolveRevision(t, f.cfg, f.repoPath, f.base.String()+"^")+ const changes = 16+ commits := make([]git.ObjectID, changes)+ for i := range commits {+ path := fmt.Sprintf("change-%d", i)+ commits[i] = gittest.WriteCommit(t, f.cfg, f.repoPath, gittest.WithParents(root), gittest.WithMessage(path),+ gittest.WithTreeEntries(rootEntry, gittest.TreeEntry{Mode: "100644", Path: path, Content: path}))+ }- errPack := errors.New("injected pack failure")- original := packBatchObjects- packBatchObjects = func(context.Context, *localrepo.Repo, string) (int, error) { return 0, errPack }- t.Cleanup(func() { packBatchObjects = original })+ attempts := make([]int32, changes)+ errs := make([]error, changes)+ var wg sync.WaitGroup+ for i := range commits {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ resp, err := landUnbatched(t, ctx, client, f, commits[i])+ errs[i] = err+ attempts[i] = resp.GetAttempts()+ }(i)+ }+ wg.Wait()- _, f, land := unbatchedLandFixture(t, ctx)+ retried := 0+ for i := range commits {+ require.NoError(t, errs[i], "land %d", i)+ if attempts[i] > 1 {+ retried+++ }+ }+ require.Positive(t, retried, "the lands raced: some lost a compare-and-swap and retried")- err := land(f.diverged1)- testhelper.RequireGrpcCode(t, err, codes.Internal)- require.ErrorContains(t, err, errPack.Error())- requireBranchAt(t, f.cfg, f.repoPath, git.DefaultBranch, f.base)+ tree := gittest.Exec(t, f.cfg, "-C", f.repoPath, "ls-tree", "--name-only", "refs/heads/"+git.DefaultBranch)+ for i := range commits {+ require.Contains(t, string(tree), fmt.Sprintf("change-%d", i))+ }+ require.Len(t, packIndexes(t, filepath.Join(f.repoPath, "objects")), packsBefore, "no attempt left a pack")+ gittest.Exec(t, f.cfg, "-C", f.repoPath, "fsck", "--connectivity-only")+}++// helperFsyncConfig returns the effective core.fsync of the running git processes whose command+// line holds every one of needles: the last -c core.fsync=... wins, as for git.+func helperFsyncConfig(t *testing.T, needles ...string) []string {+ t.Helper()++ procs, err := filepath.Glob("/proc/[0-9]*/cmdline")+ require.NoError(t, err)++ var values []string+ for _, proc := range procs {+ raw, err := os.ReadFile(proc)+ if err != nil {+ continue+ }+ args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")+ line := strings.Join(args, " ")++ match := true+ for _, needle := range needles {+ if !strings.Contains(line, needle) {+ match = false+ break+ }+ }+ if !match {+ continue+ }++ value := ""+ for i := 0; i+1 < len(args); i++ {+ if args[i] == "-c" {+ if v, ok := strings.CutPrefix(args[i+1], "core.fsync="); ok {+ value = v+ }+ }+ }+ values = append(values, value)+ }+ return values+}++// TestLandingAttemptModesSetTheHelperFsync checks the command lines of the helper processes that an+// attempt starts: in the pack mode the merge-tree and hash-object helpers run with core.fsync=none,+// and in the loose mode with the server's configuration, which fsyncs objects.+func TestLandingAttemptModesSetTheHelperFsync(t *testing.T) {+ t.Parallel()++ ctx := featureflag.ContextWithFeatureFlag(testhelper.Context(t), featureflag.GPGSigning, false)++ for _, tc := range []struct {+ mode landPublishMode+ want func(t *testing.T, value string)+ }{+ {mode: landPublishPack, want: func(t *testing.T, value string) { require.Equal(t, "none", value) }},+ {mode: landPublishLoose, want: func(t *testing.T, value string) { require.Contains(t, value, "objects") }},+ } {+ t.Run(fmt.Sprint(tc.mode), func(t *testing.T) {+ cfg := testcfg.Build(t)+ srv, cfg := runLandBatchStepperServer(t, cfg)+ f := newStepperFixture(t, ctx, cfg)++ attempt, cleanup, err := srv.newLandingAttempt(ctx, f.repoProto, tc.mode, ignoreLandPhase)+ require.NoError(t, err)+ defer cleanup()++ // A rebase starts both helpers, which run until the attempt ends.+ step, err := attempt.decide(ctx, f.base, f.request(t, ctx, f.diverged1))+ require.NoError(t, err)+ require.True(t, step.rewritten)++ for _, helper := range []string{"merge-tree", "hash-object"} {+ values := helperFsyncConfig(t, f.repoPath, helper, "--stdin")+ require.Len(t, values, 1, "one %s helper of the attempt is running", helper)+ tc.want(t, values[0])+ }+ })+ }}diff --git a/internal/gitaly/service/operations/land_batch_pack.go b/internal/gitaly/service/operations/land_batch_pack.goindex 458146b..1b3e457 100644--- a/internal/gitaly/service/operations/land_batch_pack.go+++ b/internal/gitaly/service/operations/land_batch_pack.go@@ -26,12 +26,13 @@ var packBatchObjects = packAttemptObjects// holds no loose object, and nothing is done.//// This is how a batch makes its objects durable once (doc/graft/PLAN8.md, section 3.2): the rebaser-// writes them without fsync, git pack-objects writes and fsyncs the pack and its index with the-// fsync configuration of the server, and the quarantine migration of the update then moves only the-// pack files, fsyncing them again, before the reference is updated. The loose files are removed only-// after git pack-objects succeeded; on any error they are left as they are, and the attempt, whose-// quarantine is discarded, publishes nothing. A file that is not a loose object, such as a pack that-// a helper wrote, is left for the migration, which fsyncs it.+// writes them without fsync, and git pack-objects writes and fsyncs the pack and its index with the+// fsync configuration of the server, so that every object file of the quarantine is durable before+// the quarantine migration of the update moves it, as a quarantine requires. The loose files are+// removed only after git pack-objects succeeded; on any error they are left as they are, and the+// attempt, whose quarantine is discarded, publishes nothing. The rebaser writes only loose objects;+// the other writers of an attempt (conflict resolution, the merge strategy) write with the fsync+// configuration of the server, so their files are durable already.//// objectDir must be a quarantine object directory that only the attempt writes to: the files are// listed once, so an object written while the pack is built would stay loose.diff --git a/internal/gitaly/service/operations/land_batch_phase4_test.go b/internal/gitaly/service/operations/land_batch_phase4_test.goindex ff18083..0c37596 100644--- a/internal/gitaly/service/operations/land_batch_phase4_test.go+++ b/internal/gitaly/service/operations/land_batch_phase4_test.go@@ -421,8 +421,8 @@ func TestLandBatch_rebaseGoesThroughRebaser(t *testing.T) {var rebases atomic.Int64original := newBatchRebaser- newBatchRebaser = func(repo *localrepo.Repo) batchRebaser {- return countingRebaser{batchRebaser: original(repo), rebases: &rebases}+ newBatchRebaser = func(repo *localrepo.Repo, mode landPublishMode) batchRebaser {+ return countingRebaser{batchRebaser: original(repo, mode), rebases: &rebases}}t.Cleanup(func() { newBatchRebaser = original })diff --git a/internal/gitaly/service/operations/land_batch_server.go b/internal/gitaly/service/operations/land_batch_server.goindex fc6515c..7435585 100644--- a/internal/gitaly/service/operations/land_batch_server.go+++ b/internal/gitaly/service/operations/land_batch_server.go@@ -179,7 +179,7 @@ func (s *Server) executeLandBatch(ctx context.Context, _ landlane.Key, reqs []*l// newLandBatchStepper starts a batch attempt on repository. Its phases are observed in// graft_land_batch_phase_seconds. The returned function ends the attempt.func (s *Server) newLandBatchStepper(ctx context.Context, repository *gitalypb.Repository) (*serverLandBatchStepper, func(), error) {- attempt, cleanup, err := s.newLandingAttempt(ctx, repository, observeLandBatchPhase)+ attempt, cleanup, err := s.newLandingAttempt(ctx, repository, landPublishPack, observeLandBatchPhase)if err != nil {return nil, nil, err}diff --git a/internal/graft/rebase/engine.go b/internal/graft/rebase/engine.goindex 554d949..be44d5b 100644--- a/internal/graft/rebase/engine.go+++ b/internal/graft/rebase/engine.go@@ -69,8 +69,10 @@ type engineConfig struct {}// WithDeferredDurability starts the helper processes without fsync of the loose objects they write.-// The caller makes those objects durable before any reference points to them. The batch stepper of-// UserLand does: it publishes the objects of its quarantine as one fsynced pack (doc/graft/PLAN8.md,+// The caller takes over their durability: a quarantine's contract is that every object file in it+// is durable before the quarantine migration moves it into the repository, so the caller must make+// the objects durable before the migration, not rely on it. The batch attempts of UserLand do: they+// pack the objects of their quarantine into one fsynced pack before the update (doc/graft/PLAN8.md,// section 3.2).func WithDeferredDurability() Option {return func(cfg *engineConfig) {