Git designed for AI agents.
PLAN11 review: land intents, branch names as bytes, and the minors
Federico Elia103807bb6f18
diff --git a/internal/git/gitcmd/hooks_payload.go b/internal/git/gitcmd/hooks_payload.goindex 361e126..e99ccc3 100644--- a/internal/git/gitcmd/hooks_payload.go+++ b/internal/git/gitcmd/hooks_payload.go@@ -167,7 +167,10 @@ type ChangelogMeta struct {// that branch (see changelog.WithLands and doc/graft/PLAN11.md). They are never cut:// WithLands bounds them by changelog.MaxLandsSize and refuses a land whose identities// do not fit. Only the server builds them from the context, like Members.- LandReference string `json:"land_reference,omitempty"`+ // LandReference holds the branch name as bytes, which JSON encodes in base64:+ // a branch name may hold bytes that are not valid UTF-8, which a JSON string+ // would replace.+ LandReference []byte `json:"land_reference,omitempty"`Lands []changelog.Land `json:"lands,omitempty"`}@@ -182,7 +185,7 @@ func (m ChangelogMeta) IsZero() bool {m.MVCCCommit == "" &&len(m.Members) == 0 &&!m.MembersTruncated &&- m.LandReference == "" &&+ len(m.LandReference) == 0 &&len(m.Lands) == 0}@@ -290,7 +293,7 @@ func NewHooksPayload(Rewritten: changelog.IsRewritten(ctx),Members: members,MembersTruncated: membersTruncated,- LandReference: landReference,+ LandReference: []byte(landReference),Lands: lands,MVCCCommit: mvccCommitHandle(ctx, cfg),},diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.goindex c3bc4dc..ac91f9e 100644--- a/internal/gitaly/hook/referencetransaction.go+++ b/internal/gitaly/hook/referencetransaction.go@@ -135,6 +135,13 @@ func (m *GitLabHookManager) ReferenceTransactionHook(ctx context.Context, stateif err := m.storagePrepared(ctx, payload, env, changes); err != nil {return err}++ // The file changelog records the identities of a land only after git committed it, so+ // they are recorded as intents now, before git may commit (doc/graft/PLAN11.md, 4.2).+ // An error aborts the land, which would otherwise be committed without its identity.+ if err := m.recordLandIntents(ctx, payload, env, changes); err != nil {+ return fmt.Errorf("recording the land intents: %w", err)+ }}if !isForceDeletionsOnly(objectHash, bytes.NewReader(changes)) {@@ -184,6 +191,44 @@ func changesetInvocation(env []string) string {return meta.Invocation}+// recordLandIntents records the land identities of a prepared transaction as intents, when the+// file changelog is the one that records them and the transaction updates the branch they landed+// on (see changelog.RecordLandIntents). In the authoritative storage mode the storage core records+// them with the reference change, in one database transaction.+func (m *GitLabHookManager) recordLandIntents(ctx context.Context, payload gitcmd.HooksPayload, env []string, changes []byte) error {+ if m.cfg.Graft.Changelog.Disabled || m.storageAuthoritative() || payload.Repo == nil {+ return nil+ }++ meta, err := gitcmd.ChangelogMetaFromEnv(env)+ if errors.Is(err, gitcmd.ErrPayloadNotFound) {+ return nil+ }+ if err != nil {+ return fmt.Errorf("reading changelog metadata: %w", err)+ }+ if len(meta.Lands) == 0 {+ return nil+ }++ updatesBranch := false+ for _, update := range changelog.ParseUpdates(changes) {+ if update.Reference == string(meta.LandReference) {+ updatesBranch = true+ }+ }+ if !updatesBranch {+ return nil+ }++ repoPath, err := m.locator.GetRepoPath(ctx, payload.Repo)+ if err != nil {+ return fmt.Errorf("resolving repository path: %w", err)+ }++ return changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Lands)+}+// prepareChangesets gives the prepared transaction its place in the order of the changeset log,// see changesetTickets.func (m *GitLabHookManager) prepareChangesets(payload gitcmd.HooksPayload, env []string, changes []byte) {diff --git a/internal/gitaly/hook/storage_voter.go b/internal/gitaly/hook/storage_voter.goindex 1c2b9dd..a554395 100644--- a/internal/gitaly/hook/storage_voter.go+++ b/internal/gitaly/hook/storage_voter.go@@ -102,7 +102,7 @@ func changelogMetaOf(payload gitcmd.HooksPayload, graftMeta gitcmd.ChangelogMetaRewritten: graftMeta.Rewritten,Members: graftMeta.Members,MembersTruncated: graftMeta.MembersTruncated,- LandReference: graftMeta.LandReference,+ LandReference: string(graftMeta.LandReference),Lands: graftMeta.Lands,}if meta.Operation == "" {diff --git a/internal/gitaly/service/operations/land.go b/internal/gitaly/service/operations/land.goindex 54f62df..cd8eccb 100644--- a/internal/gitaly/service/operations/land.go+++ b/internal/gitaly/service/operations/land.go@@ -466,6 +466,13 @@ func validateUserLandRequest(ctx context.Context, locator storage.Locator, in *greturn fmt.Errorf("invalid branch name: %w", err)}+ // The land records its identity in its update (doc/graft/PLAN11.md), which bounds the name of+ // the branch. The request is refused here rather than its update, so that it fails alone, also+ // in a batch.+ if err := changelog.CheckLandReference("refs/heads/" + string(in.GetBranch())); err != nil {+ return err+ }+if in.GetCommitId() == "" {return errors.New("empty commit id")}diff --git a/internal/gitaly/service/operations/land_batch.go b/internal/gitaly/service/operations/land_batch.goindex 4897636..39bbfd9 100644--- a/internal/gitaly/service/operations/land_batch.go+++ b/internal/gitaly/service/operations/land_batch.go@@ -323,6 +323,10 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequestrewritten := falsevar members []intvar changes []changelog.Member+ // landedAs maps the commit of each member to the commit it landed as in the running tip, so+ // that a later request of the same commit, a retry that joined the batch of its first land,+ // is answered as already landed.+ landedAs := map[git.ObjectID]git.ObjectID{}for pos, i := range idx {req := reqs[i]outs[pos].OldTip = cur@@ -349,6 +353,14 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequeste.deliverEarly(ctx, i, len(reqs), outs[pos], final)}+ case step.tip == cur && landedAs[step.commit] != "":+ // An earlier member of the batch landed the same commit: the request is a retry+ // of it, and is answered as already landed with the commit of that member. The+ // answer holds only if the update of the batch does: the request comes after a+ // member, so when the update does not hold it is landed again (see run).+ outs[pos].LandedCommit = landedAs[step.commit]+ outs[pos].AlreadyLanded = true+case step.tip == cur:// The commit became empty on top of the running tip and was dropped. The request// reports a landing that changed nothing.@@ -360,6 +372,7 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequestmembers = append(members, pos)changes = append(changes, memberOf(req, step))+ landedAs[step.commit] = step.tipouts[pos].LandedCommit = step.tipouts[pos].Rewritten = step.rewrittendiff --git a/internal/gitaly/service/operations/land_identity.go b/internal/gitaly/service/operations/land_identity.goindex aecc04d..6b31413 100644--- a/internal/gitaly/service/operations/land_identity.go+++ b/internal/gitaly/service/operations/land_identity.go@@ -5,6 +5,7 @@ import ("context""errors""fmt"+ "slices""strings""github.com/feder-cr/git-ai/internal/git"@@ -29,9 +30,14 @@ type landIdentity struct {revertedBy git.ObjectID}-// detail explains the answer, see UserLandResponse.already_landed_detail.+// detail explains the answer, see UserLandResponse.already_landed_detail. A land known from its+// intent has no changelog sequence: git committed it and the changelog did not record it, or has not+// yet.func (id *landIdentity) detail() string {detail := fmt.Sprintf("landed as %s at changelog %d", id.landed, id.sequence)+ if id.sequence == 0 {+ detail = fmt.Sprintf("landed as %s, which the changelog does not record", id.landed)+ }if id.revertedBy != "" {detail += fmt.Sprintf("; it was reverted later by %s", id.revertedBy)}@@ -60,44 +66,49 @@ func landWithRecord(ctx context.Context, reference git.ReferenceName, lands []ch// landedEarlier returns the earlier land of commitID on the branch reference when it is still on// the branch, that is when the commit it landed as is reachable from tip; nil otherwise. A change// whose commit is not reachable from tip may have landed as a rewritten commit, and the record of-// that land is the only way to tell (doc/graft/PLAN11.md, section 5). Only then is the commit that-// reverted it looked for, to explain the answer.+// that land is the only way to tell (doc/graft/PLAN11.md, section 5). The lookup gives the land the+// changelog recorded first, then the intents of lands it does not record; the first on the branch+// answers. An intent of an update that was not committed is not on the branch, and answers+// nothing. Only once a land answers is the commit that reverted it looked for, to explain the+// answer.func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.ReferenceName, tip, commitID git.ObjectID) (*landIdentity, error) {- record, found, err := a.server.lookupLand(ctx, a.repository, reference, commitID)+ records, err := a.server.lookupLand(ctx, a.repository, reference, commitID)if err != nil {return nil, err}- if !found {- return nil, nil- }- landed, err := a.objectHash.FromHex(record.Landed)- if err != nil {- return nil, structerr.NewInternal("land identity of %s: %w", commitID, err)- }+ for _, record := range records {+ landed, err := a.objectHash.FromHex(record.Landed)+ if err != nil {+ return nil, structerr.NewInternal("land identity of %s: %w", commitID, err)+ }- onBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())- if err != nil {- // The branch was reset past the earlier land and its commit was pruned since: the change- // is no longer on the branch.- if errors.As(err, new(localrepo.InvalidCommitError)) {- return nil, nil+ onBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())+ if err != nil {+ // The commit does not exist: an intent of an update that was not committed, whose+ // objects were never migrated, or a land whose branch was reset past it and whose+ // commit was pruned since. Either way the change is not on the branch through it.+ if errors.As(err, new(localrepo.InvalidCommitError)) {+ continue+ }+ return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)+ }+ if !onBranch {+ continue}- return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)- }- if !onBranch {- return nil, nil- }- // The revert only explains the answer, so a failure to look for it does not change the- // answer: it is logged, and the answer names no revert.- revertedBy, err := a.revertOf(ctx, landed, tip)- if err != nil {- a.server.logger.WithError(err).WithField("landed", landed.String()).- Error("graft: looking for the revert of an earlier land failed; the answer names no revert")+ // The revert only explains the answer, so a failure to look for it does not change the+ // answer: it is logged, and the answer names no revert.+ revertedBy, err := a.revertOf(ctx, landed, tip)+ if err != nil {+ a.server.logger.WithError(err).WithField("landed", landed.String()).+ Error("graft: looking for the revert of an earlier land failed; the answer names no revert")+ }++ return &landIdentity{landed: landed, sequence: record.Sequence, revertedBy: revertedBy}, nil}- return &landIdentity{landed: landed, sequence: record.Sequence, revertedBy: revertedBy}, nil+ return nil, nil}// revertSearchCommits bounds the commits that revertOf reads: the revert is looked for among the@@ -106,10 +117,10 @@ func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.Referevar revertSearchCommits = 1000// revertOf returns the latest commit after landed up to tip whose message holds "This reverts-// commit <landed>", the line that git revert and graft-verify write, or an empty ID when there is-// none among the revertSearchCommits commits closest to tip. A revert written another way, or-// further from the tip, is not found. The revert only explains the answer: the land is already-// landed whatever revertOf finds.+// commit <landed>", the line that git revert and graft-verify write, and that no later commit+// reverts in turn, or an empty ID when there is none among the revertSearchCommits commits closest+// to tip. A revert written another way, or further from the tip, is not found. The revert only+// explains the answer: the land is already landed whatever revertOf finds.func (a *landingAttempt) revertOf(ctx context.Context, landed, tip git.ObjectID) (git.ObjectID, error) {var stdout, stderr bytes.Bufferif err := a.repo.ExecAndWait(ctx, gitcmd.Command{@@ -124,59 +135,87 @@ func (a *landingAttempt) revertOf(ctx context.Context, landed, tip git.ObjectID)}// The output is pairs of fields, the commit ID and its message, each ended by a NUL, newest- // commit first. Messages cannot hold a NUL.- line := "This reverts commit " + landed.String()+ // commit first. Messages cannot hold a NUL. The commits that a newer commit reverts are+ // collected on the way, so that a revert that was reverted in turn is passed over.+ revertedLater := map[string]bool{}fields := strings.Split(stdout.String(), "\x00")for i := 0; i+1 < len(fields); i += 2 {- if !strings.Contains(fields[i+1], line) {- continue+ commit := strings.TrimSpace(fields[i])+ reverts := revertedCommits(fields[i+1], a.objectHash.EncodedLen())++ if slices.Contains(reverts, landed.String()) && !revertedLater[commit] {+ revertedBy, err := a.objectHash.FromHex(commit)+ if err != nil {+ return "", fmt.Errorf("looking for a revert of %s: %w", landed, err)+ }+ return revertedBy, nil}- revertedBy, err := a.objectHash.FromHex(strings.TrimSpace(fields[i]))- if err != nil {- return "", fmt.Errorf("looking for a revert of %s: %w", landed, err)+ for _, reverted := range reverts {+ revertedLater[reverted] = true}- return revertedBy, nil}return "", nil}-// lookupLand returns the last land of original on the branch reference of repository that the-// changelog recorded, from where the changelog lives: the storage core's database in its-// authoritative mode, the file log of the repository otherwise. Without a changelog nothing is-// recorded, and nothing is found.-func (s *Server) lookupLand(ctx context.Context, repository *gitalypb.Repository, reference git.ReferenceName, original git.ObjectID) (changelog.LandRecord, bool, error) {+// revertedCommits returns the commits that message says it reverts: the object IDs of length+// hexLength that follow "This reverts commit ".+func revertedCommits(message string, hexLength int) []string {+ const prefix = "This reverts commit "++ var reverted []string+ for {+ start := strings.Index(message, prefix)+ if start < 0 {+ return reverted+ }+ message = message[start+len(prefix):]+ if len(message) >= hexLength {+ reverted = append(reverted, message[:hexLength])+ }+ }+}++// lookupLand returns what is known of the lands of original on the branch reference of repository,+// from where the changelog lives: in the storage core's authoritative mode, the land its database+// recorded with the reference change, in one transaction; otherwise the land the file log recorded+// and the land intents it does not record (see changelog.LookupLand). Without a changelog nothing+// is recorded, and nothing is found.+func (s *Server) lookupLand(ctx context.Context, repository *gitalypb.Repository, reference git.ReferenceName, original git.ObjectID) ([]changelog.LandRecord, error) {if s.graftStorage.Authoritative() {repo, err := s.graftStorage.Meta.GetRepository(ctx, corestore.KeyOf(repository))if err != nil {// A repository that the database does not track has no recorded land. Whether it// can be written to is the write path's decision, as for any land.if errors.Is(err, corestore.ErrRepositoryNotFound) {- return changelog.LandRecord{}, false, nil+ return nil, nil}- return changelog.LandRecord{}, false, structerr.NewUnavailable("storage core: %w", err)+ return nil, structerr.NewUnavailable("storage core: %w", err)}record, found, err := s.graftStorage.Meta.LookupLand(ctx, repo.ID, reference.String(), original.String())if err != nil {- return changelog.LandRecord{}, false, structerr.NewUnavailable("storage core: %w", err)+ return nil, structerr.NewUnavailable("storage core: %w", err)}- return record, found, nil+ if !found {+ return nil, nil+ }+ return []changelog.LandRecord{record}, nil}if s.graft.Changelog.Disabled {- return changelog.LandRecord{}, false, nil+ return nil, nil}repoPath, err := s.locator.GetRepoPath(ctx, repository)if err != nil {- return changelog.LandRecord{}, false, err+ return nil, err}- record, found, err := changelog.LookupLand(repoPath, reference.String(), original.String())+ records, err := changelog.LookupLand(repoPath, reference.String(), original.String())if err != nil {- return changelog.LandRecord{}, false, structerr.NewInternal("looking up the land identity: %w", err)+ return nil, structerr.NewInternal("looking up the land identity: %w", err)}- return record, found, nil+ return records, nil}diff --git a/internal/gitaly/service/operations/land_identity_test.go b/internal/gitaly/service/operations/land_identity_test.goindex 152b4ea..68039a5 100644--- a/internal/gitaly/service/operations/land_identity_test.go+++ b/internal/gitaly/service/operations/land_identity_test.go@@ -2,13 +2,21 @@ package operationsimport ("context"+ "errors"+ "fmt"+ "os"+ "strconv"+ "strings""testing""github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config""github.com/feder-cr/git-ai/internal/graft/changelog"+ "github.com/feder-cr/git-ai/internal/testhelper")// TestLandBatch_retryAfterALostResponse lands two changes by rebase in one batch, and then lands@@ -73,6 +81,208 @@ func testLandBatchRetryAfterALostResponse(t *testing.T, ctx context.Context) {requireBranchAt(t, cfg, repoPath, git.DefaultBranch, tip)}+// TestUserLand_intentAnswersWhatTheChangelogMisses lands a change whose update git committed and+// whose changelog entry is lost, as a crash between the two leaves it (doc/graft/PLAN11.md, 4.2):+// the intent the hook recorded before git committed answers the retry.+func TestUserLand_intentAnswersWhatTheChangelogMisses(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testUserLandIntentAnswersWhatTheChangelogMisses)+}++func testUserLandIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.Context) {+ t.Parallel()++ ctx, cfg, client := setupOperationsService(t, ctx)+ fx := newRetryFixture(t, ctx, cfg)++ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)+ first, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ landed := git.ObjectID(first.GetLandedCommitId())++ // The changelog lost the entry of the land, its only entry.+ require.NoError(t, os.Remove(changelog.Path(fx.repoPath)))++ second, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ require.True(t, second.GetAlreadyLanded())+ require.Equal(t, landed.String(), second.GetLandedCommitId())+ require.Zero(t, second.GetLandedSequence())+ require.Equal(t, fmt.Sprintf("landed as %s, which the changelog does not record", landed), second.GetAlreadyLandedDetail())+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, landed)+}++// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing records intents whose updates git never+// committed, as an update that was aborted after its prepared phase leaves them: their landed+// commit is not on the branch, so the change lands.+func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testUserLandIntentOfAnUncommittedUpdateAnswersNothing)+}++func testUserLandIntentOfAnUncommittedUpdateAnswersNothing(t *testing.T, ctx context.Context) {+ t.Parallel()++ ctx, cfg, client := setupOperationsService(t, ctx)+ fx := newRetryFixture(t, ctx, cfg)++ // One intent names a commit that does not exist, the other one that is not on the branch.+ require.NoError(t, changelog.RecordLandIntents(fx.repoPath, git.DefaultRef.String(), []changelog.Land{+ {Original: fx.change.String(), Landed: strings.Repeat("1", len(fx.change))},+ {Original: fx.change.String(), Landed: fx.change.String()},+ }))++ resp, err := client.UserLand(ctx, landRequest(fx.repoProto, git.DefaultBranch, fx.change))+ require.NoError(t, err)+ require.False(t, resp.GetAlreadyLanded())+ require.True(t, resp.GetRewritten())+}++// TestUserLand_identityOfABranchWhoseNameIsNotUTF8 lands a change by rebase on a branch whose name+// is not valid UTF-8, through the hooks: its identity is recorded with the name byte for byte, and+// the retry answers from it.+func TestUserLand_identityOfABranchWhoseNameIsNotUTF8(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testUserLandIdentityOfABranchWhoseNameIsNotUTF8)+}++func testUserLandIdentityOfABranchWhoseNameIsNotUTF8(t *testing.T, ctx context.Context) {+ t.Parallel()++ ctx, cfg, client := setupOperationsService(t, ctx)+ fx := newRetryFixture(t, ctx, cfg)+ branch := "f\xffo"+ gittest.Exec(t, cfg, "-C", fx.repoPath, "update-ref", "refs/heads/"+branch, fx.tip.String())++ request := landRequest(fx.repoProto, branch, fx.change)+ first, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ require.True(t, first.GetRewritten())+ landed := git.ObjectID(first.GetLandedCommitId())++ entries, err := changelog.List(fx.repoPath, 0, 100)+ require.NoError(t, err)+ require.Len(t, entries, 1)+ require.Equal(t, "refs/heads/"+branch, entries[0].Reference)+ require.Equal(t, []changelog.Land{{Original: fx.change.String(), Landed: landed.String()}}, entries[0].Lands)++ second, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ requireLandedEarlier(t, second, entries[0], landed, "")+}++// TestUserLand_concurrentRetries lands the same change twice at once, as a retry that reaches the+// server while the first call still runs: the change lands once, and the other call answers+// already landed with the commit it landed as.+func TestUserLand_concurrentRetries(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testUserLandConcurrentRetries)+}++func testUserLandConcurrentRetries(t *testing.T, ctx context.Context) {+ t.Parallel()++ for _, tc := range []struct {+ desc string+ graft config.Graft+ }{+ {desc: "unbatched", graft: config.Graft{}},+ {desc: "batched", graft: agentBatchingGraft(nil)},+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ ctx, cfg, client := setupAgentLandService(t, ctx, tc.graft)+ for round := 0; round < 5; round++ {+ fx := newRetryFixture(t, ctx, cfg)+ before := gittest.Exec(t, cfg, "-C", fx.repoPath, "rev-list", "--count", git.DefaultBranch)++ responses, errs := landInOrder(ctx, client, fx.repoProto,+ agentLanding{commit: fx.change},+ agentLanding{commit: fx.change},+ )+ require.NoError(t, errs[0])+ require.NoError(t, errs[1])++ landed := 0+ for _, resp := range responses {+ if !resp.GetAlreadyLanded() {+ landed+++ require.True(t, resp.GetRewritten())+ }+ }+ require.Equal(t, 1, landed, "round %d: the change lands once", round)+ require.Equal(t, responses[0].GetLandedCommitId(), responses[1].GetLandedCommitId(), "round %d", round)++ tip := gittest.ResolveRevision(t, cfg, fx.repoPath, git.DefaultRef.String())+ require.Equal(t, responses[0].GetLandedCommitId(), tip.String(), "round %d", round)+ require.Equal(t, mustAtoi(t, before)+1, mustAtoi(t, gittest.Exec(t, cfg, "-C", fx.repoPath, "rev-list", "--count", git.DefaultBranch)), "round %d", round)+ }+ })+ }+}++// mustAtoi parses the decimal number of a git output.+func mustAtoi(t *testing.T, output []byte) int {+ t.Helper()++ n, err := strconv.Atoi(strings.TrimSpace(string(output)))+ require.NoError(t, err)+ return n+}++// TestUserLand_branchNameTooLongForTheIdentity refuses, when it arrives, a land on a branch whose+// name is too long for the identity of its update to be recorded.+func TestUserLand_branchNameTooLongForTheIdentity(t *testing.T) {+ t.Parallel()++ ctx, cfg, client := setupOperationsService(t, testhelper.Context(t))+ fx := newRetryFixture(t, ctx, cfg)++ branch := strings.TrimSuffix(strings.Repeat(strings.Repeat("a", 200)+"/", 35), "/")+ _, err := client.UserLand(ctx, landRequest(fx.repoProto, branch, fx.change))+ testhelper.RequireGrpcCode(t, err, codes.InvalidArgument)+ require.ErrorContains(t, err, "is too long to record the identity of a land on it")+}++// TestLandBatchExecutorRetryInTheBatchOfItsFirstLand puts a request and its retry in one batch:+// the retry answers already landed with the commit of the first, and only when the batch's update+// holds.+func TestLandBatchExecutorRetryInTheBatchOfItsFirstLand(t *testing.T) {+ t.Parallel()++ t.Run("the update holds", func(t *testing.T) {+ fake := &fakeLandStepper{tip: "main0"}+ outcomes := newFakeLandExecutor(fake, 3).Execute(context.Background(), testLandRequests(false, "c1", "c1"))++ require.NoError(t, outcomes[0].Err)+ require.False(t, outcomes[0].AlreadyLanded)+ require.NoError(t, outcomes[1].Err)+ require.True(t, outcomes[1].AlreadyLanded)+ require.Equal(t, git.ObjectID("c1"), outcomes[1].LandedCommit)+ require.Len(t, fake.updates, 1)+ require.Equal(t, []string{"c1"}, fake.updates[0].members2Commits(), "the retry is not a member")+ })++ t.Run("the update fails", func(t *testing.T) {+ // The first land fails with the update. Its retry is not answered as landed: it lands+ // again, alone, onto the real tip.+ fake := &fakeLandStepper{tip: "main0", script: []scriptedUpdate{{kind: landUpdateFailed, err: errors.New("disk full")}}}+ outcomes := newFakeLandExecutor(fake, 3).Execute(context.Background(), testLandRequests(false, "c1", "c1"))++ require.ErrorContains(t, outcomes[0].Err, "disk full")+ require.NoError(t, outcomes[1].Err)+ require.False(t, outcomes[1].AlreadyLanded)+ require.Equal(t, git.ObjectID("c1"), outcomes[1].LandedCommit)+ require.Len(t, fake.updates, 2)+ require.Equal(t, []string{"c1"}, fake.updates[1].members2Commits())+ })+}+// TestLandBatch_retryOfABatchOfOne retries a change that landed alone in its lane, which publishes// it as a land of its own (UpdateSolo): its identity is recorded, and the retry answers from it.func TestLandBatch_retryOfABatchOfOne(t *testing.T) {diff --git a/internal/gitaly/service/operations/land_retry_test.go b/internal/gitaly/service/operations/land_retry_test.goindex bc4e456..b7c330f 100644--- a/internal/gitaly/service/operations/land_retry_test.go+++ b/internal/gitaly/service/operations/land_retry_test.go@@ -238,6 +238,35 @@ func testUserLandRetryAfterALostResponse(t *testing.T, ctx context.Context) {}})+ t.Run("after a revert that was reverted in turn", func(t *testing.T) {+ t.Parallel()++ // The change landed, a commit reverted it, and a later commit reverted the revert: the+ // change is back on the branch, and the answer names no revert.+ fx := newRetryFixture(t, ctx, cfg)+ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)+ first, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ landed := git.ObjectID(first.GetLandedCommitId())++ reverted := gittest.WriteCommit(t, cfg, fx.repoPath,+ gittest.WithBranch(git.DefaultBranch),+ gittest.WithParents(landed),+ gittest.WithTreeEntries(fx.withoutY...),+ gittest.WithMessage(fmt.Sprintf("Revert \"y\"\n\nThis reverts commit %s.\n", landed)),+ )+ gittest.WriteCommit(t, cfg, fx.repoPath,+ gittest.WithBranch(git.DefaultBranch),+ gittest.WithParents(reverted),+ gittest.WithTreeEntries(append(fx.withoutY, gittest.TreeEntry{Mode: "100644", Path: "y", Content: "y"})...),+ gittest.WithMessage(fmt.Sprintf("Revert \"Revert \"y\"\"\n\nThis reverts commit %s.\n", reverted)),+ )++ second, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ requireLandedEarlier(t, second, landEntry(t, fx.repoPath, landed), landed, "")+ })+t.Run("after the branch was reset past the land", func(t *testing.T) {t.Parallel()diff --git a/internal/graft/changelog/lands.go b/internal/graft/changelog/lands.goindex 973b52c..c677397 100644--- a/internal/graft/changelog/lands.go+++ b/internal/graft/changelog/lands.go@@ -2,6 +2,7 @@ package changelogimport ("context"+ "encoding/base64""encoding/json""fmt""strings"@@ -32,22 +33,49 @@ const (// MaxLandsSize bounds the encoded size of the identities of one update (the reference and its// lands), which travel in the hooks payload, in the environment of every git process of the// update, next to the members (see gitcmd.ChangelogMeta). Unlike the members, the identities are- // never cut: an update whose identities do not fit fails before it runs (WithLands). With the- // limits of the server it cannot happen: a batch has at most MaxLandsPerUpdate members, a land- // of a SHA-256 repository encodes in 156 bytes, and a branch name of 4096 bytes, which no files- // reference backend exceeds, encodes in at most 6 bytes per byte: 40 KiB hold both with room.- MaxLandsSize = 40 << 10+ // never cut: a land request whose branch name could not fit is refused when it arrives+ // (CheckLandReference), and WithLands checks the bound again before the update runs. A batch+ // has at most MaxLandsPerUpdate members and a land of a SHA-256 repository encodes in 156+ // bytes, 15.6 KB for a full batch; the branch name is encoded in base64, so 24 KiB hold a full+ // batch and a branch name of 4096 bytes, which no files reference backend exceeds, with room.+ MaxLandsSize = 24 << 10// MaxLandsPerUpdate is the most lands that one update records: the largest batch the// configuration allows (config.MaxGraftLandBatchMaxSize).MaxLandsPerUpdate = 100)-// encodedLands is the encoded form of the identities of one update whose size MaxLandsSize bounds.+// encodedLands is the encoded form of the identities of one update whose size MaxLandsSize bounds,+// as gitcmd.ChangelogMeta encodes them: the reference as bytes, in base64.type encodedLands struct {- Reference string `json:"land_reference"`+ Reference []byte `json:"land_reference"`Lands []Land `json:"lands"`}+// fullBatchLandsSize is the encoded size of the identities of a full batch of SHA-256 lands on a+// branch with an empty name, the largest identities of any branch but its name.+var fullBatchLandsSize = func() int {+ lands := make([]Land, MaxLandsPerUpdate)+ for i := range lands {+ lands[i] = Land{Original: strings.Repeat("f", 64), Landed: strings.Repeat("f", 64)}+ }+ encoded, err := json.Marshal(encodedLands{Lands: lands})+ if err != nil {+ panic(fmt.Sprintf("encoding the identities of a full batch: %v", err))+ }+ return len(encoded)+}()++// CheckLandReference checks, when a land request arrives, that the identities of any update of+// reference fit MaxLandsSize: those of a full batch of SHA-256 lands on it. A branch whose name is+// too long for them cannot be landed on, since a land would not record its identity.+func CheckLandReference(reference string) error {+ if size := fullBatchLandsSize + base64.StdEncoding.EncodedLen(len(reference)); size > MaxLandsSize {+ return fmt.Errorf("the name of branch %q is too long to record the identity of a land on it: %d bytes, at most %d",+ reference, len(reference), (MaxLandsSize-fullBatchLandsSize)/4*3)+ }+ return nil+}+// ValidateLands checks the identities that an update of reference records: reference is a branch,// every land names two full object IDs, there are at most MaxLandsPerUpdate of them and their// encoding fits MaxLandsSize.@@ -68,7 +96,7 @@ func ValidateLands(reference string, lands []Land) error {}}- encoded, err := json.Marshal(encodedLands{Reference: reference, Lands: lands})+ encoded, err := json.Marshal(encodedLands{Reference: []byte(reference), Lands: lands})if err != nil {return fmt.Errorf("land identities of %q: %w", reference, err)}diff --git a/internal/graft/changelog/lands_index.go b/internal/graft/changelog/lands_index.goindex aae0e68..222e12a 100644--- a/internal/graft/changelog/lands_index.go+++ b/internal/graft/changelog/lands_index.go@@ -10,23 +10,35 @@ import ("io""os""path/filepath"+ "sort""sync"+ "sync/atomic")const (// landsFileName is the name of the index of the land identities inside dirName.landsFileName = "changelog.lands"// maxIdleLandsIndexes bounds the number of indexes that are kept in memory while no lookup- // uses them. An index holds every land of its repository, so fewer of them are kept than of- // the states of the logs (maxIdleStates). A dropped index is loaded from its file again.+ // uses them. A dropped index is loaded from its file again.maxIdleLandsIndexes = 64+ // maxIdleLandsBytes bounds the memory of the indexes that are kept while no lookup uses them,+ // as estimated by landsIndex.bytes. The least recently used ones beyond it are dropped, so an+ // index larger than the bound is dropped as soon as its lookup ends.+ maxIdleLandsBytes = 256 << 20+ // compactLandsMinSize is the size from which an index file is compacted once most of its+ // records are superseded by later ones.+ compactLandsMinSize = 1 << 20+ // compactIntentsMinSize is the size from which an intents file is compacted, see+ // compactIntents.+ compactIntentsMinSize = 64 << 10)// The index of the land identities of a repository (doc/graft/PLAN11.md, 4.2) maps a branch and an// original commit to the last changelog entry that landed that commit on that branch. It is derived// from the changelog: the log entries carry the lands (Entry.Lands), and the index is the result of// reading them, kept in memory and in the file graft/changelog.lands so that a new process does not-// read the whole log again.+// read the whole log again. A lookup also reads the land intents (see landIntent), which cover the+// lands whose update git committed before the changelog recorded it.//// The file is a sequence of blocks of JSON lines. A block holds the records of the lands of some// log entries, in log order, and ends with a mark: the sequence of the last log entry the block@@ -41,6 +53,10 @@ const (// entry is not the one the mark names (the log was replaced, for example by a repository created// again at the same path) the index is rebuilt from the whole log. The file is therefore written// without fsync: what a crash leaves of it is either a good prefix or is cut off.+//+// In memory, the index also knows the log file it was built from: a log file that is another file+// (another inode) rebuilds it, and a lookup whose log ends with the entry of the mark, in the same+// file, reads nothing.// landsIndexKey identifies a land in an index.type landsIndexKey struct {@@ -60,29 +76,55 @@ func (m landsMark) matches(entry Entry) bool {return entry.Sequence == m.sequence && entry.Reference == m.reference && entry.NewOID == m.newOID}+// describes reports whether head is the head of the log that the mark names the last entry of.+func (m landsMark) describes(head logHead) bool {+ return head.sequence == m.sequence && head.reference == m.reference && head.newOID == m.newOID+}+// landsIndex is the in-process view of the index of one repository.type landsIndex struct {// mu serializes the lookups of this process in the repository and guards the fields below.mu sync.Mutex- // users counts the callers that hold the index. It is guarded by landsIndexesMu.- users int+ // users and lastUsed are guarded by landsIndexesMu: the callers that hold the index, and the+ // tick of its last release, which orders the eviction.+ users int+ lastUsed uint64+ // bytes is the estimate of the memory of the index, see estimate. It is read by the eviction+ // without mu.+ bytes atomic.Int64+// loaded is false until the index has been read from its file.loaded bool// mark is the log entry up to which records is complete; its sequence is zero when records// covers no entry.mark landsMark+ // logDevice and logInode identify the log file that records was read from.+ logDevice, logInode uint64// records maps the lands of the log entries up to mark to their last record.records map[landsIndexKey]LandRecord+ // recordLines is the number of record lines of the index file, superseded ones included.+ recordLines int// fileSize is the size of the index file when this process last read or wrote it. A file of// another size was written by another process and is read again.fileSize int64++ // intents are the land intents read from the intents file, by land, and intentsFile what is+ // known of that file.+ intents map[landsIndexKey][]landIntent+ intentsFile intentsFile+ // intentsKept is the size of the intents file after its last compaction by this process. The+ // file is compacted again once it doubled, so that intents that cannot go yet are not+ // rewritten by every lookup.+ intentsKept int64}var (landsIndexesMu sync.Mutex// landsIndexes holds the indexes by the path of their file. An index is kept after its last- // user releases it, up to maxIdleLandsIndexes idle ones.+ // user releases it, within maxIdleLandsIndexes and maxIdleLandsBytes.landsIndexes = map[string]*landsIndex{}+ // landsTick orders the releases of the indexes. It is guarded by landsIndexesMu.+ landsTick uint64)// acquireLandsIndex returns the index whose file is path, and the function that releases it.@@ -101,32 +143,45 @@ func acquireLandsIndex(path string) (*landsIndex, func()) {defer landsIndexesMu.Unlock()index.users--+ landsTick+++ index.lastUsed = landsTickevictIdleLandsIndexes()}}-// evictIdleLandsIndexes drops idle indexes until at most maxIdleLandsIndexes remain idle. The caller-// must hold landsIndexesMu.+// evictIdleLandsIndexes drops the least recently used idle indexes until at most+// maxIdleLandsIndexes of them, of at most maxIdleLandsBytes together, remain. The caller must hold+// landsIndexesMu.func evictIdleLandsIndexes() {- if len(landsIndexes) <= maxIdleLandsIndexes {- return+ type idleIndex struct {+ path string+ lastUsed uint64+ bytes int64}- idle := 0- for _, index := range landsIndexes {+ var (+ idle []idleIndex+ total int64+ )+ for path, index := range landsIndexes {if index.users == 0 {- idle+++ bytes := index.bytes.Load()+ idle = append(idle, idleIndex{path: path, lastUsed: index.lastUsed, bytes: bytes})+ total += bytes}}+ if len(idle) <= maxIdleLandsIndexes && total <= maxIdleLandsBytes {+ return+ }- for path, index := range landsIndexes {- if idle <= maxIdleLandsIndexes {+ sort.Slice(idle, func(a, b int) bool { return idle[a].lastUsed < idle[b].lastUsed })+ for _, index := range idle {+ if len(idle) <= maxIdleLandsIndexes && total <= maxIdleLandsBytes {return}- if index.users == 0 {- delete(landsIndexes, path)- idle--- }+ delete(landsIndexes, index.path)+ idle = idle[1:]+ total -= index.bytes}}@@ -136,12 +191,14 @@ func landsPath(repoPath string) string {return filepath.Join(repoPath, dirName, landsFileName)}-// LookupLand returns the last land of original on reference that the changelog of the repository-// whose git directory is repoPath recorded (doc/graft/PLAN11.md): the sequence of its entry and the-// commit it landed as. found is false when the changelog recorded no land of original on reference.-// The lookup reads the durable entries of the log (see AppendGroup) that the index does not cover-// yet, and adds them to the index first.-func LookupLand(repoPath, reference, original string) (record LandRecord, found bool, err error) {+// LookupLand returns what the repository whose git directory is repoPath knows of the lands of+// original on reference (doc/graft/PLAN11.md): first the last land that the changelog recorded,+// with the sequence of its entry and the commit it landed as, then the land intents (see+// landIntent) of lands that the changelog does not record, with a zero sequence. A land answers+// only when its landed commit is on the branch, which the caller checks. The result is empty when+// nothing is known. The lookup reads the durable entries of the log (see AppendGroup) and the+// intents that the index does not cover yet, and adds them to the index first.+func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {index, release := acquireLandsIndex(landsPath(repoPath))defer release()@@ -151,26 +208,71 @@ func LookupLand(repoPath, reference, original string) (record LandRecord, foundif err := index.catchUp(repoPath); err != nil {// What the failed catch-up did to the index is not known, so the next lookup reads it again.index.loaded = false- return LandRecord{}, false, err+ index.intentsFile = intentsFile{}+ return nil, err+ }++ key := landsIndexKey{reference: reference, original: original}+ var lands []LandRecord+ confirmed, found := index.records[key]+ if found {+ lands = append(lands, confirmed)+ }+ for _, intent := range index.intents[key] {+ if found && (intent.Landed == confirmed.Landed || confirmed.Sequence > intent.Head) {+ continue+ }+ lands = append(lands, LandRecord{Landed: intent.Landed})}- record, found = index.records[landsIndexKey{reference: reference, original: original}]- return record, found, nil+ return lands, nil}-// catchUp brings the index up to the last durable entry of the log. The caller must hold i.mu.+// catchUp brings the index up to the last durable entry of the log and the intents up to the end+// of their file, and compacts their files when they grew large. The caller must hold i.mu.func (i *landsIndex) catchUp(repoPath string) error {- last, err := LastSequence(repoPath)+ if err := i.catchUpLog(repoPath); err != nil {+ return err+ }+ if err := i.catchUpIntents(repoPath); err != nil {+ return err+ }+ i.bytes.Store(i.estimate())++ return nil+}++// estimate returns an estimate of the memory of the index: its strings and the overhead of maps+// and slices. The caller must hold i.mu.+func (i *landsIndex) estimate() int64 {+ const overhead = 96+ var bytes int64+ for key, record := range i.records {+ bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)) + overhead+ }+ for key, intents := range i.intents {+ for _, intent := range intents {+ bytes += int64(len(key.reference)+len(key.original)+len(intent.Landed)+len(intent.Reference)) + overhead+ }+ }+ return bytes+}++// catchUpLog brings the index up to the last durable entry of the log. The caller must hold i.mu.+func (i *landsIndex) catchUpLog(repoPath string) error {+ head, err := readLogHead(repoPath)if err != nil {return err}- if last == 0 {+ if head.sequence == 0 {// A log without entries recorded no land. The index, in memory or in its file, is of// another log if it holds anything, and is rebuilt once this log has entries.i.reset(0)return nil}- if i.loaded && last == i.mark.sequence {++ sameLog := i.loaded && head.device == i.logDevice && head.inode == i.logInode+ if sameLog && i.mark.describes(head) {return nil}@@ -195,13 +297,20 @@ func (i *landsIndex) catchUp(repoPath string) error {if err != nil {return fmt.Errorf("stat land index: %w", err)}- if !i.loaded || info.Size() != i.fileSize {++ block, err := landsBlock{}, errLandsIndexStale+ switch {+ case i.loaded && !sameLog:+ // The log is another file than the one the index was read from: the repository was+ // created again. The index is rebuilt.+ case !i.loaded || info.Size() != i.fileSize:if err := i.load(f); err != nil {return err}+ block, err = i.scanLog(repoPath)+ default:+ block, err = i.scanLog(repoPath)}-- block, err := i.scanLog(repoPath)if errors.Is(err, errLandsIndexStale) {// The log is not the one the index was built from: build it again from the whole log.if err := truncateFile(f, 0); err != nil {@@ -214,25 +323,109 @@ func (i *landsIndex) catchUp(repoPath string) error {return err}- if block.mark.sequence == i.mark.sequence {- i.loaded = true- return nil+ if block.mark.sequence != i.mark.sequence {+ encoded, err := block.encode()+ if err != nil {+ return err+ }+ if _, err := f.WriteAt(encoded, i.fileSize); err != nil {+ return fmt.Errorf("writing land index: %w", err)+ }++ for _, record := range block.records {+ i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}+ }+ i.recordLines += len(block.records)+ i.mark = block.mark+ i.fileSize += int64(len(encoded))}+ i.logDevice, i.logInode = head.device, head.inode+ i.loaded = true++ if i.fileSize >= compactLandsMinSize && i.recordLines > 2*len(i.records) {+ return i.compact(path)+ }++ return nil+}++// compact rewrites the index file with only the records that are not superseded, as one block,+// and replaces the old file with it. The caller must hold i.mu and the lock file of the index; the+// rename makes the other processes read the new file.+func (i *landsIndex) compact(path string) error {+ block := landsBlock{mark: i.mark}+ for key, record := range i.records {+ block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed})+ }+ sort.Slice(block.records, func(a, b int) bool { return block.records[a].sequence < block.records[b].sequence })+encoded, err := block.encode()if err != nil {return err}- if _, err := f.WriteAt(encoded, i.fileSize); err != nil {- return fmt.Errorf("writing land index: %w", err)++ tmp := path + ".compact"+ if err := os.WriteFile(tmp, encoded, 0o644); err != nil {+ return fmt.Errorf("compacting land index: %w", err)+ }+ if err := os.Rename(tmp, path); err != nil {+ return fmt.Errorf("compacting land index: %w", err)}- for _, record := range block.records {- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}+ i.recordLines = len(block.records)+ i.fileSize = int64(len(encoded))++ return nil+}++// catchUpIntents reads the land intents recorded since the last lookup, and compacts the intents+// file when it grew large. The caller must hold i.mu.+func (i *landsIndex) catchUpIntents(repoPath string) error {+ path := intentsPath(repoPath)++ add := func(intent landIntent) {+ key := landsIndexKey{reference: string(intent.Reference), original: intent.Original}+ i.intents[key] = append(i.intents[key], intent)}- i.mark = block.mark- i.fileSize += int64(len(encoded))- i.loaded = true++ if i.intents == nil {+ i.intents = map[landsIndexKey][]landIntent{}+ }+ known, reset, err := readIntents(path, i.intentsFile, add)+ if err != nil {+ return err+ }+ if reset {+ // The file is not the one that was read before, and the intents read before are not+ // its: they are dropped, and the file is read again from its start.+ i.intents = map[landsIndexKey][]landIntent{}+ if known, _, err = readIntents(path, intentsFile{}, add); err != nil {+ return err+ }+ }+ i.intentsFile = known++ if known.offset < compactIntentsMinSize || known.offset < 2*i.intentsKept {+ return nil+ }++ // An intent may go once the changelog confirms it (a record of its land with its landed+ // commit) or supersedes it (a later record of the same original commit on the branch).+ if err := compactIntents(repoPath, path, func(intent landIntent) bool {+ record, found := i.records[landsIndexKey{reference: string(intent.Reference), original: intent.Original}]+ return found && (record.Landed == intent.Landed || record.Sequence > intent.Head)+ }); err != nil {+ return err+ }++ i.intents = map[landsIndexKey][]landIntent{}+ known, _, err = readIntents(path, intentsFile{}, add)+ if err != nil {+ return err+ }+ i.intentsFile = known+ i.intentsKept = known.offsetreturn nil}@@ -350,6 +543,7 @@ func (b landsBlock) encode() ([]byte, error) {func (i *landsIndex) reset(fileSize int64) {i.mark = landsMark{}i.records = map[landsIndexKey]LandRecord{}+ i.recordLines = 0i.fileSize = fileSizei.loaded = false}@@ -408,6 +602,7 @@ func (i *landsIndex) load(f *os.File) error {for _, record := range pending {i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}}+ i.recordLines += len(pending)i.mark = landsMark{sequence: decoded.Mark, reference: string(decoded.Reference), newOID: decoded.NewOID}i.fileSize = offsetpending = nildiff --git a/internal/graft/changelog/lands_intents.go b/internal/graft/changelog/lands_intents.gonew file mode 100644index 0000000..772b8f8--- /dev/null+++ b/internal/graft/changelog/lands_intents.go@@ -0,0 +1,259 @@+package changelog++import (+ "bufio"+ "bytes"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "io/fs"+ "os"+ "path/filepath"+ "sync"+ "syscall"+)++// intentsFileName is the name of the file of the land intents inside dirName.+const intentsFileName = "changelog.intents"++// The land intents of a repository (doc/graft/PLAN11.md, 4.2) close the window between git+// committing the update of a land and the changelog recording it. The changelog entry of an update,+// which carries its lands, is written in the committed phase of the reference transaction, after+// git committed the references: a crash, a full disk or a lock that times out in between leaves the+// branch with the land and the changelog without its identity. So, in the prepared phase, before+// git may commit, the hook records the lands of the update as intents and fsyncs them. A lookup+// answers from the confirmed records of the index and from the intents; the land path counts an+// intent only when its landed commit is on the branch, so the intent of an update that was not+// committed answers nothing.+//+// The file is JSON lines, appended under the lock file of the intents and fsync'd before the+// prepared phase ends. Unlike the index it is not derived data. An intent is dropped by a+// compaction once the changelog confirms it (a record of its land with its landed commit) or+// supersedes it (a later record of the same original commit on the branch); the intent of an+// update that was never committed and never landed again stays.++// landIntent is one line of the intents file.+type landIntent struct {+ // Reference is the branch, as bytes (base64 in JSON): branch names may hold bytes that are+ // not valid UTF-8.+ Reference []byte `json:"ref"`+ Original string `json:"original"`+ Landed string `json:"landed"`+ // Head is the sequence of the last durable changelog entry when the intent was recorded. The+ // entry of the land, if any, comes after it, so a record of the same commit at a later+ // sequence confirms or supersedes the intent.+ Head uint64 `json:"head"`+}++// intentsPath returns the path of the intents file of the repository whose git directory is+// repoPath.+func intentsPath(repoPath string) string {+ return filepath.Join(repoPath, dirName, intentsFileName)+}++// intentDirsSynced holds the intents files of this process whose directory entries are durable,+// see RecordLandIntents.+var intentDirsSynced sync.Map++// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the+// update of reference about to be committed lands lands (see the comment above landIntent). It+// returns once the intents are fsync'd; an error must abort the update, which would otherwise be+// committed without its identity.+func RecordLandIntents(repoPath, reference string, lands []Land) error {+ head, err := readLogHead(repoPath)+ if err != nil {+ return err+ }++ path := intentsPath(repoPath)+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {+ return fmt.Errorf("creating changelog directory: %w", err)+ }++ unlock, err := lockFile(path + lockSuffix)+ if err != nil {+ return err+ }+ defer unlock()++ f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)+ if err != nil {+ return fmt.Errorf("opening land intents: %w", err)+ }+ defer f.Close()++ // The bytes after the last newline are a torn line of a writer that crashed before its fsync,+ // whose update was therefore never committed. They are cut off, or the next line would be+ // joined to them.+ end, err := endOfLastLine(f)+ if err != nil {+ return err+ }++ var buf bytes.Buffer+ for _, land := range lands {+ line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Head: head.sequence})+ if err != nil {+ return fmt.Errorf("encoding land intent: %w", err)+ }+ buf.Write(line)+ buf.WriteByte('\n')+ }++ if err := truncateFile(f, end); err != nil {+ return fmt.Errorf("truncating land intents: %w", err)+ }+ if _, err := f.WriteAt(buf.Bytes(), end); err != nil {+ return fmt.Errorf("writing land intents: %w", err)+ }+ if err := syncFile(f); err != nil {+ return fmt.Errorf("syncing land intents: %w", err)+ }++ // The intents are durable only once the directory entries of their file are, which a writer+ // that created the file may have crashed before syncing: the first write of this process+ // syncs them.+ if _, synced := intentDirsSynced.Load(path); !synced {+ if err := syncLogDirs(repoPath, path); err != nil {+ return err+ }+ intentDirsSynced.Store(path, struct{}{})+ }++ return nil+}++// endOfLastLine returns the offset just after the last newline of f, 0 when it has none.+func endOfLastLine(f *os.File) (int64, error) {+ info, err := f.Stat()+ if err != nil {+ return 0, fmt.Errorf("stat land intents: %w", err)+ }++ nl, err := lastNewlineBefore(f, info.Size())+ if err != nil {+ return 0, err+ }++ return nl + 1, nil+}++// intentsFile is what a reader of an intents file knows of it.+type intentsFile struct {+ // device and inode identify the file. A compaction replaces it.+ device, inode uint64+ // offset is the end of the complete lines read so far.+ offset int64+}++// readIntents reads the complete lines of the intents file at path after known, calling add for+// each intent. It returns what the reader knows of the file after the read. When the file is not+// the one known (it was replaced) or is shorter (it was replaced or cut), it is read from the+// start and reset is true: the intents read before do not describe it any more. A missing file has+// no intents.+func readIntents(path string, known intentsFile, add func(landIntent)) (intentsFile, bool, error) {+ f, err := os.Open(path)+ if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ return intentsFile{}, known != (intentsFile{}), nil+ }+ return known, false, fmt.Errorf("opening land intents: %w", err)+ }+ defer f.Close()++ info, err := f.Stat()+ if err != nil {+ return known, false, fmt.Errorf("stat land intents: %w", err)+ }+ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return known, false, fmt.Errorf("stat land intents: no file identity in %T", info.Sys())+ }++ current := intentsFile{device: uint64(stat.Dev), inode: stat.Ino, offset: known.offset}+ reset := current.device != known.device || current.inode != known.inode || info.Size() < known.offset+ if reset {+ current.offset = 0+ }++ r := bufio.NewReader(io.NewSectionReader(f, current.offset, info.Size()-current.offset))+ for {+ line, err := r.ReadBytes('\n')+ if err != nil && !errors.Is(err, io.EOF) {+ return known, false, fmt.Errorf("reading land intents: %w", err)+ }+ // A line without its newline is still being written, or is the torn line of a writer that+ // crashed: it is read once it is complete, or cut off by the next writer.+ if len(line) == 0 || line[len(line)-1] != '\n' {+ break+ }+ current.offset += int64(len(line))++ // A line that does not decode is not an intent. Only a writer that crashed before its+ // fsync leaves one, and its update was not committed.+ var intent landIntent+ if json.Unmarshal(line, &intent) != nil || !isObjectID(intent.Original) || !isObjectID(intent.Landed) {+ continue+ }+ add(intent)+ }++ return current, reset, nil+}++// compactIntents rewrites the intents file at path without the intents that drop says may go,+// under the lock file of the intents, so that no intent recorded meanwhile is lost: the file is+// read whole under the lock. The new file is fsync'd and renamed over the old one, and the+// directory is fsync'd, before the lock is released.+func compactIntents(repoPath, path string, drop func(landIntent) bool) error {+ unlock, err := lockFile(path + lockSuffix)+ if err != nil {+ return err+ }+ defer unlock()++ var (+ kept bytes.Buffer+ encodeErr error+ )+ if _, _, err := readIntents(path, intentsFile{}, func(intent landIntent) {+ if drop(intent) || encodeErr != nil {+ return+ }+ line, err := json.Marshal(intent)+ if err != nil {+ encodeErr = fmt.Errorf("encoding land intent: %w", err)+ return+ }+ kept.Write(line)+ kept.WriteByte('\n')+ }); err != nil {+ return err+ }+ if encodeErr != nil {+ return encodeErr+ }++ tmp := path + ".compact"+ f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)+ if err != nil {+ return fmt.Errorf("compacting land intents: %w", err)+ }+ if _, err := f.Write(kept.Bytes()); err != nil {+ f.Close()+ return fmt.Errorf("compacting land intents: %w", err)+ }+ if err := syncFile(f); err != nil {+ f.Close()+ return fmt.Errorf("compacting land intents: %w", err)+ }+ if err := f.Close(); err != nil {+ return fmt.Errorf("compacting land intents: %w", err)+ }+ if err := os.Rename(tmp, path); err != nil {+ return fmt.Errorf("compacting land intents: %w", err)+ }++ return syncLogDirs(repoPath, path)+}diff --git a/internal/graft/changelog/lands_intents_test.go b/internal/graft/changelog/lands_intents_test.gonew file mode 100644index 0000000..f42268a--- /dev/null+++ b/internal/graft/changelog/lands_intents_test.go@@ -0,0 +1,325 @@+package changelog++import (+ "bytes"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+)++// recordIntent records the intent of a land of original on reference as landed.+func recordIntent(t *testing.T, repoPath, reference, original, landed string) {+ t.Helper()+ require.NoError(t, RecordLandIntents(repoPath, reference, []Land{{Original: original, Landed: landed}}))+}++// intentsOf returns the intents in the intents file of the repository.+func intentsOf(t *testing.T, repoPath string) []landIntent {+ t.Helper()++ var intents []landIntent+ _, _, err := readIntents(intentsPath(repoPath), intentsFile{}, func(intent landIntent) {+ intents = append(intents, intent)+ })+ require.NoError(t, err)+ return intents+}++func TestLookupLandAnswersFromIntents(t *testing.T) {+ t.Parallel()++ t.Run("an intent without a log", func(t *testing.T) {+ // Git committed the first land of the repository, and the process stopped before the+ // changelog recorded it.+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))++ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/other", landOID(1)))+ })++ t.Run("an intent that the changelog confirms", func(t *testing.T) {+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))++ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ })++ t.Run("an intent that a later land supersedes", func(t *testing.T) {+ // The update of the intent was not committed, and the change was landed again.+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(3))++ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(3)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ })++ t.Run("an intent after the recorded land", func(t *testing.T) {+ // The change landed, the branch was reset past it, and its new land was committed+ // without the changelog recording it: both are known, the record first.+ repoPath := t.TempDir()+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))++ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}, {Landed: landOID(3)}},+ lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ })++ t.Run("a branch whose name is not valid UTF-8", func(t *testing.T) {+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/\xff", landOID(1), landOID(2))++ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/\xff", landOID(1)))+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/�", landOID(1)))+ })++ t.Run("intents recorded by another process", func(t *testing.T) {+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ require.Len(t, lookupAll(t, repoPath, "refs/heads/main", landOID(1)), 1)++ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))+ require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))+ })+}++func TestLandIntentsFile(t *testing.T) {+ t.Parallel()++ t.Run("a torn line is cut off by the next intent", func(t *testing.T) {+ // A writer crashed in the middle of its line, before its fsync: its update was not+ // committed. The next writer cuts the line off, so that its own line stays whole.+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ appendToFile(t, intentsPath(repoPath), `{"ref":"cmVmcy9oZWFkcy9tYWlu","original":"`+landOID(5))++ _, err := LookupLand(repoPath, "refs/heads/main", landOID(5))+ require.NoError(t, err, "a reader ignores the torn line")++ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))+ require.Len(t, intentsOf(t, repoPath), 2)+ require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/main", landOID(5)))+ })++ t.Run("a line that is not an intent is skipped", func(t *testing.T) {+ repoPath := t.TempDir()+ require.NoError(t, os.MkdirAll(filepath.Dir(intentsPath(repoPath)), 0o755))+ require.NoError(t, os.WriteFile(intentsPath(repoPath), []byte("\x00\x00\n{\"ref\":\"\",\"original\":\"nope\",\"landed\":\"x\"}\n"), 0o644))+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))++ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ })++ t.Run("an intent records the head of the log", func(t *testing.T) {+ repoPath := t.TempDir()+ seq := appendPlain(t, repoPath, "refs/heads/main", landOID(9))+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))++ intents := intentsOf(t, repoPath)+ require.Len(t, intents, 1)+ require.Equal(t, seq, intents[0].Head)+ })+}++// TestLandIntentsAreDurable replaces syncFile, so it does not run in parallel.+func TestLandIntentsAreDurable(t *testing.T) {+ t.Run("the intents are fsync'd before they return", func(t *testing.T) {+ repoPath := t.TempDir()+ var synced []string+ replaceSyncFile(t, func(f *os.File) error {+ synced = append(synced, filepath.Base(f.Name()))+ return f.Sync()+ })++ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ require.Contains(t, synced, intentsFileName)+ })++ t.Run("a failed fsync fails the intent", func(t *testing.T) {+ repoPath := t.TempDir()+ replaceSyncFile(t, func(f *os.File) error {+ if filepath.Base(f.Name()) == intentsFileName {+ return fmt.Errorf("injected")+ }+ return f.Sync()+ })++ require.ErrorContains(t, RecordLandIntents(repoPath, "refs/heads/main", []Land{{Original: landOID(1), Landed: landOID(2)}}), "injected")+ })+}++func TestLandIntentsCompaction(t *testing.T) {+ t.Parallel()++ repoPath := t.TempDir()++ // Enough intents to compact the file: half of them confirmed by the changelog, a quarter+ // superseded by a later land, a quarter never committed and never landed again.+ const n = 2000+ lands := make([]Land, n)+ for i := range lands {+ lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}+ }+ require.NoError(t, RecordLandIntents(repoPath, "refs/heads/main", lands))+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))++ var confirmed []Land+ for i := 0; i < n/2; i++ {+ confirmed = append(confirmed, lands[i])+ }+ for i := n / 2; i < 3*n/4; i++ {+ confirmed = append(confirmed, Land{Original: lands[i].Original, Landed: landOID(30000 + i)})+ }+ updates := make([]Update, 0, len(confirmed))+ for i := range confirmed {+ updates = append(updates, Update{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)})+ }+ // One entry per land, each carrying its own land.+ records := make([]Record, len(confirmed))+ for i, land := range confirmed {+ records[i] = Record{Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{land}}, Updates: updates[i : i+1]}+ }+ _, err := AppendGroup(repoPath, records)+ require.NoError(t, err)++ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))++ kept := intentsOf(t, repoPath)+ require.Len(t, kept, n/4, "only the intents that the changelog neither confirms nor supersedes are kept")+ for _, intent := range kept {+ require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)+ }++ // The kept intents, more than the size from which a file is compacted, are not rewritten by+ // every lookup.+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))+ compacted, err := os.Stat(intentsPath(repoPath))+ require.NoError(t, err)+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))+ again, err := os.Stat(intentsPath(repoPath))+ require.NoError(t, err)+ require.True(t, os.SameFile(compacted, again))++ // Every land is still answered as before the compaction, also by a new process.+ forgetLandsIndex(t, repoPath)+ for i, land := range lands {+ records := lookupAll(t, repoPath, "refs/heads/main", land.Original)+ require.Len(t, records, 1)+ switch {+ case i < n/2:+ require.Equal(t, land.Landed, records[0].Landed)+ require.NotZero(t, records[0].Sequence)+ case i < 3*n/4:+ require.Equal(t, landOID(30000+i), records[0].Landed)+ default:+ require.Equal(t, LandRecord{Landed: land.Landed}, records[0])+ }+ }+}++func TestLandsIndexCompaction(t *testing.T) {+ t.Parallel()++ // The same commit lands again and again (its branch reset past it each time): every record+ // but the last is superseded.+ repoPath := t.TempDir()+ const n = 8000+ records := make([]Record, n)+ for i := range records {+ records[i] = Record{+ Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{{Original: landOID(1), Landed: landOID(100 + i)}}},+ Updates: []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(100 + i)}},+ }+ }+ entries, err := AppendGroup(repoPath, records)+ require.NoError(t, err)+ last := entries[n-1][0]++ record, found := lookup(t, repoPath, "refs/heads/main", landOID(1))+ require.True(t, found)+ require.Equal(t, LandRecord{Sequence: last.Sequence, Landed: landOID(100 + n - 1)}, record)++ lines := landsIndexLines(t, repoPath)+ require.Len(t, lines, 2, "the compacted file holds one record and its mark")+ require.Equal(t, last.Sequence, lines[1].Mark)++ forgetLandsIndex(t, repoPath)+ record, found = lookup(t, repoPath, "refs/heads/main", landOID(1))+ require.True(t, found)+ require.Equal(t, LandRecord{Sequence: last.Sequence, Landed: landOID(100 + n - 1)}, record)+}++func TestLandsIndexOfAReplacedLogFile(t *testing.T) {+ t.Parallel()++ // The repository is created again at the same path, and its log ends with the same entry as+ // the old one: the index in memory, whose mark still matches, must not answer for it.+ repoPath := t.TempDir()+ appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ appendPlain(t, repoPath, "refs/heads/main", landOID(9))+ _, found := lookup(t, repoPath, "refs/heads/main", landOID(1))+ require.True(t, found)++ entries, err := List(repoPath, 0, 10)+ require.NoError(t, err)+ entries[0].Lands = nil+ var buf bytes.Buffer+ for _, entry := range entries {+ line, err := json.Marshal(entry)+ require.NoError(t, err)+ buf.Write(line)+ buf.WriteByte('\n')+ }+ replacement := Path(repoPath) + ".new"+ require.NoError(t, os.WriteFile(replacement, buf.Bytes(), 0o644))+ require.NoError(t, os.Rename(replacement, Path(repoPath)))+ forgetLog(t, repoPath)++ _, found = lookup(t, repoPath, "refs/heads/main", landOID(1))+ require.False(t, found)+}++func TestLandsIndexesAreEvictedLeastRecentlyUsedFirst(t *testing.T) {+ // The test fills the shared cache of indexes, so it does not run in parallel.+ landsIndexesMu.Lock()+ saved := landsIndexes+ landsIndexes = map[string]*landsIndex{}+ landsIndexesMu.Unlock()+ t.Cleanup(func() {+ landsIndexesMu.Lock()+ landsIndexes = saved+ landsIndexesMu.Unlock()+ })++ repos := make([]string, maxIdleLandsIndexes+1)+ for i := range repos {+ repos[i] = t.TempDir()+ appendLand(t, repos[i], "refs/heads/main", landOID(1), landOID(2))+ lookup(t, repos[i], "refs/heads/main", landOID(1))+ }+ cached := func(repoPath string) bool {+ landsIndexesMu.Lock()+ defer landsIndexesMu.Unlock()+ _, ok := landsIndexes[landsPath(repoPath)]+ return ok+ }++ // The first repository was used least recently when the last one came in.+ require.False(t, cached(repos[0]))+ for _, repoPath := range repos[1:] {+ require.True(t, cached(repoPath))+ }++ // Using the second repository again keeps it; the third is now the least recently used.+ lookup(t, repos[1], "refs/heads/main", landOID(1))+ lookup(t, repos[0], "refs/heads/main", landOID(1))+ require.True(t, cached(repos[1]))+ require.False(t, cached(repos[2]))+}diff --git a/internal/graft/changelog/lands_test.go b/internal/graft/changelog/lands_test.goindex 89b31c0..96c1403 100644--- a/internal/graft/changelog/lands_test.go+++ b/internal/graft/changelog/lands_test.go@@ -40,13 +40,25 @@ func appendPlain(t *testing.T, repoPath, reference, newOID string) uint64 {return entries[0].Sequence}-// lookup looks up a land and fails the test on an error.-func lookup(t *testing.T, repoPath, reference, original string) (LandRecord, bool) {+// lookupAll looks up the lands of original on reference and fails the test on an error.+func lookupAll(t *testing.T, repoPath, reference, original string) []LandRecord {t.Helper()- record, found, err := LookupLand(repoPath, reference, original)+ records, err := LookupLand(repoPath, reference, original)require.NoError(t, err)- return record, found+ return records+}++// lookup returns the land of original on reference that the changelog recorded.+func lookup(t *testing.T, repoPath, reference, original string) (LandRecord, bool) {+ t.Helper()++ for _, record := range lookupAll(t, repoPath, reference, original) {+ if record.Sequence != 0 {+ return record, true+ }+ }+ return LandRecord{}, false}// forgetLandsIndex drops the in-memory index of the repository, as a new process starts without it.@@ -77,14 +89,21 @@ func TestValidateLands(t *testing.T) {})t.Run("the largest batch with the longest branch name fits", func(t *testing.T) {- // Every byte of the name is escaped by JSON into 6 bytes, the worst case.- reference := "refs/heads/" + strings.Repeat("<", 4096-len("refs/heads/"))+ // The name travels in base64; bytes that are not valid UTF-8 cost no more than others.+ reference := "refs/heads/" + strings.Repeat("\xff<", (4096-len("refs/heads/"))/2)lands := make([]Land, MaxLandsPerUpdate)for i := range lands {lands[i] = Land{Original: sha256OID(2 * i), Landed: sha256OID(2*i + 1)}}require.NoError(t, ValidateLands(reference, lands))+ require.NoError(t, CheckLandReference(reference))+ })++ t.Run("a branch name too long for a full batch is refused when the land arrives", func(t *testing.T) {+ longest := "refs/heads/" + strings.Repeat("a", (MaxLandsSize-fullBatchLandsSize)/4*3-len("refs/heads/"))+ require.NoError(t, CheckLandReference(longest))+ require.ErrorContains(t, CheckLandReference(longest+"aaaa"), "is too long to record the identity of a land on it")})t.Run("more lands than a batch has", func(t *testing.T) {@@ -97,7 +116,7 @@ func TestValidateLands(t *testing.T) {})t.Run("identities beyond the size bound", func(t *testing.T) {- reference := "refs/heads/" + strings.Repeat("<", MaxLandsSize/6)+ reference := "refs/heads/" + strings.Repeat("<", MaxLandsSize/4*3)require.ErrorContains(t, ValidateLands(reference, []Land{land}), fmt.Sprintf("at most %d", MaxLandsSize))})diff --git a/internal/graft/changelog/log.go b/internal/graft/changelog/log.goindex c101eb8..661cdb2 100644--- a/internal/graft/changelog/log.go+++ b/internal/graft/changelog/log.go@@ -206,6 +206,10 @@ type logState struct {loaded bool// lastSeq is the sequence of the last valid entry.lastSeq uint64+ // lastReference and lastNewOID are the reference and the new value of the last valid entry,+ // empty when there is none. With lastSeq they identify the last entry (see LogHead).+ lastReference string+ lastNewOID string// validEnd is the offset just after the last complete, valid entry.validEnd int64// fileSize is the size of the file when the cache was last updated.@@ -452,6 +456,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entriesvar (buf bytes.Bufferpoints []indexPoint+ last Entry)seq := s.lastSeqfor i, record := range records {@@ -491,6 +496,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entriesbuf.Write(line)buf.WriteByte('\n')recorded = append(recorded, entry)+ last = entry}entries[i] = recorded}@@ -520,6 +526,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entries}s.lastSeq = seq+ s.lastReference, s.lastNewOID = last.Reference, last.NewOIDs.validEnd += int64(buf.Len())s.fileSize = s.validEnds.loaded = true@@ -606,6 +613,7 @@ func (s *logState) load(f *os.File, size int64) error {func (s *logState) rescan(f *os.File, size int64) error {s.rescans++s.lastSeq = 0+ s.lastReference, s.lastNewOID = "", ""s.validEnd = 0// The bytes after the last newline are a torn tail, which is not part of the@@ -635,6 +643,7 @@ func (s *logState) rescan(f *os.File, size int64) error {var entry Entryif err := json.Unmarshal(line, &entry); err == nil && entry.Sequence != 0 {s.lastSeq = entry.Sequence+ s.lastReference, s.lastNewOID = entry.Reference, entry.NewOIDs.validEnd = nl + 1break}@@ -760,7 +769,7 @@ var truncateFile = func(f *os.File, size int64) error {}// durableView returns the end of the durable entries of the log at path and the-// sequence of the last of them. f is the log file, opened by the caller. Readers+// last of them (see logHead). f is the log file, opened by the caller. Readers// read the log only up to that end.//// The cached state only moves past entries once they are durable, so while it@@ -772,46 +781,46 @@ var truncateFile = func(f *os.File, size int64) error {// page cache are durable once a reader may see them. So are the directory entries// of the log, which that writer may have crashed before syncing (see// syncLogDirs), unless this process synced them already.-func (s *logState) durableView(repoPath, path string, f *os.File) (int64, uint64, error) {+func (s *logState) durableView(repoPath, path string, f *os.File) (int64, logHead, error) {s.mu.Lock()defer s.mu.Unlock()info, err := f.Stat()if err != nil {- return 0, 0, fmt.Errorf("stat changelog: %w", err)+ return 0, logHead{}, fmt.Errorf("stat changelog: %w", err)}if valid, err := s.cacheDescribes(f, info.Size()); err != nil {- return 0, 0, err+ return 0, logHead{}, err} else if valid && s.dirsSynced {- return s.validEnd, s.lastSeq, nil+ return s.validEnd, s.head(), nil}unlock, err := lockFile(path + lockSuffix)if err != nil {- return 0, 0, err+ return 0, logHead{}, err}defer unlock()if err := syncFile(f); err != nil {- return 0, 0, fmt.Errorf("syncing changelog: %w", err)+ return 0, logHead{}, fmt.Errorf("syncing changelog: %w", err)}if !s.dirsSynced {if err := syncLogDirs(repoPath, path); err != nil {- return 0, 0, err+ return 0, logHead{}, err}s.dirsSynced = true}info, err = f.Stat()if err != nil {- return 0, 0, fmt.Errorf("stat changelog: %w", err)+ return 0, logHead{}, fmt.Errorf("stat changelog: %w", err)}if err := s.load(f, info.Size()); err != nil {s.loaded = false- return 0, 0, err+ return 0, logHead{}, err}- return s.validEnd, s.lastSeq, nil+ return s.validEnd, s.head(), nil}// syncDir fsyncs the directory dir, so that the entries it holds are durable.@@ -953,6 +962,37 @@ func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {// AppendGroup). It uses the cached state of the log while it describes the file,// so its cost does not depend on the size of the log.func LastSequence(repoPath string) (uint64, error) {+ head, err := readLogHead(repoPath)+ if err != nil {+ return 0, err+ }++ return head.sequence, nil+}++// logHead identifies the last durable entry of a log and the file that holds it.+type logHead struct {+ // sequence, reference and newOID are those of the last durable entry; the zero value when the+ // log has none.+ sequence uint64+ reference string+ newOID string+ // device and inode identify the log file. A log that was replaced, for example with its+ // repository, is another file.+ device uint64+ inode uint64+}++// head returns the head of the log as the cache describes it, without the file identity. The+// caller must hold s.mu.+func (s *logState) head() logHead {+ return logHead{sequence: s.lastSeq, reference: s.lastReference, newOID: s.lastNewOID}+}++// readLogHead returns the head of the log of the repository whose git directory is repoPath: the+// zero value when the log does not exist. Like LastSequence, it uses the cached state of the log+// while it describes the file.+func readLogHead(repoPath string) (logHead, error) {path := Path(repoPath)state, release := acquireState(path)@@ -961,18 +1001,28 @@ func LastSequence(repoPath string) (uint64, error) {f, err := os.Open(path)if err != nil {if errors.Is(err, fs.ErrNotExist) {- return 0, nil+ return logHead{}, nil}- return 0, fmt.Errorf("opening changelog: %w", err)+ return logHead{}, fmt.Errorf("opening changelog: %w", err)}defer f.Close()- _, lastSeq, err := state.durableView(repoPath, path, f)+ info, err := f.Stat()if err != nil {- return 0, err+ return logHead{}, fmt.Errorf("stat changelog: %w", err)+ }+ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return logHead{}, fmt.Errorf("stat changelog: no file identity in %T", info.Sys())+ }++ _, head, err := state.durableView(repoPath, path, f)+ if err != nil {+ return logHead{}, err}+ head.device, head.inode = uint64(stat.Dev), stat.Ino- return lastSeq, nil+ return head, nil}// List returns up to limit entries with a sequence greater than after, indiff --git a/sdk/land.go b/sdk/land.goindex a044e2d..d949018 100644--- a/sdk/land.go+++ b/sdk/land.go@@ -83,6 +83,14 @@ type LandOptions struct {// the update lands the change; with ResolveConflicts it may run the conflict// resolver again, a model included, for the same conflict, and LandResult// reports the retries (UnavailableRetries), so that this cost is visible.+//+// The record of the first land needs a server released after v0.2.0 with its+// changeset log on ([graft.changelog] not disabled). Against an older server,+// or one whose changeset log is disabled, the retry of a lost LandRebase replays+// the change: every commit of it comes out empty and the branch is left+// unchanged, without AlreadyLanded, unless a later commit reverted the change in+// between, which the retry then lands again. Set SkipRebaseLands for such+// servers.type UnavailableRetry struct {// Disabled turns the retries off: Land returns Unavailable at once.Disabled bool