Skip to content

Git designed for AI agents.

PLAN11 review: land intents, branch names as bytes, and the minors

Federico Elia103807bb6f18
diff --git a/internal/git/gitcmd/hooks_payload.go b/internal/git/gitcmd/hooks_payload.go
index 361e126..e99ccc3 100644
--- a/internal/git/gitcmd/hooks_payload.go
+++ b/internal/git/gitcmd/hooks_payload.go
@@ -167,7 +167,10 @@ type ChangelogMeta struct {
// that branch (see changelog.WithLands and doc/graft/PLAN11.md). They are never cut:
// WithLands bounds them by changelog.MaxLandsSize and refuses a land whose identities
// do not fit. Only the server builds them from the context, like Members.
- LandReference string `json:"land_reference,omitempty"`
+ // LandReference holds the branch name as bytes, which JSON encodes in base64:
+ // a branch name may hold bytes that are not valid UTF-8, which a JSON string
+ // would replace.
+ LandReference []byte `json:"land_reference,omitempty"`
Lands []changelog.Land `json:"lands,omitempty"`
}
@@ -182,7 +185,7 @@ func (m ChangelogMeta) IsZero() bool {
m.MVCCCommit == "" &&
len(m.Members) == 0 &&
!m.MembersTruncated &&
- m.LandReference == "" &&
+ len(m.LandReference) == 0 &&
len(m.Lands) == 0
}
@@ -290,7 +293,7 @@ func NewHooksPayload(
Rewritten: changelog.IsRewritten(ctx),
Members: members,
MembersTruncated: membersTruncated,
- LandReference: landReference,
+ LandReference: []byte(landReference),
Lands: lands,
MVCCCommit: mvccCommitHandle(ctx, cfg),
},
diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.go
index c3bc4dc..ac91f9e 100644
--- a/internal/gitaly/hook/referencetransaction.go
+++ b/internal/gitaly/hook/referencetransaction.go
@@ -135,6 +135,13 @@ func (m *GitLabHookManager) ReferenceTransactionHook(ctx context.Context, state
if err := m.storagePrepared(ctx, payload, env, changes); err != nil {
return err
}
+
+ // The file changelog records the identities of a land only after git committed it, so
+ // they are recorded as intents now, before git may commit (doc/graft/PLAN11.md, 4.2).
+ // An error aborts the land, which would otherwise be committed without its identity.
+ if err := m.recordLandIntents(ctx, payload, env, changes); err != nil {
+ return fmt.Errorf("recording the land intents: %w", err)
+ }
}
if !isForceDeletionsOnly(objectHash, bytes.NewReader(changes)) {
@@ -184,6 +191,44 @@ func changesetInvocation(env []string) string {
return meta.Invocation
}
+// recordLandIntents records the land identities of a prepared transaction as intents, when the
+// file changelog is the one that records them and the transaction updates the branch they landed
+// on (see changelog.RecordLandIntents). In the authoritative storage mode the storage core records
+// them with the reference change, in one database transaction.
+func (m *GitLabHookManager) recordLandIntents(ctx context.Context, payload gitcmd.HooksPayload, env []string, changes []byte) error {
+ if m.cfg.Graft.Changelog.Disabled || m.storageAuthoritative() || payload.Repo == nil {
+ return nil
+ }
+
+ meta, err := gitcmd.ChangelogMetaFromEnv(env)
+ if errors.Is(err, gitcmd.ErrPayloadNotFound) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("reading changelog metadata: %w", err)
+ }
+ if len(meta.Lands) == 0 {
+ return nil
+ }
+
+ updatesBranch := false
+ for _, update := range changelog.ParseUpdates(changes) {
+ if update.Reference == string(meta.LandReference) {
+ updatesBranch = true
+ }
+ }
+ if !updatesBranch {
+ return nil
+ }
+
+ repoPath, err := m.locator.GetRepoPath(ctx, payload.Repo)
+ if err != nil {
+ return fmt.Errorf("resolving repository path: %w", err)
+ }
+
+ return changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Lands)
+}
+
// prepareChangesets gives the prepared transaction its place in the order of the changeset log,
// see changesetTickets.
func (m *GitLabHookManager) prepareChangesets(payload gitcmd.HooksPayload, env []string, changes []byte) {
diff --git a/internal/gitaly/hook/storage_voter.go b/internal/gitaly/hook/storage_voter.go
index 1c2b9dd..a554395 100644
--- a/internal/gitaly/hook/storage_voter.go
+++ b/internal/gitaly/hook/storage_voter.go
@@ -102,7 +102,7 @@ func changelogMetaOf(payload gitcmd.HooksPayload, graftMeta gitcmd.ChangelogMeta
Rewritten: graftMeta.Rewritten,
Members: graftMeta.Members,
MembersTruncated: graftMeta.MembersTruncated,
- LandReference: graftMeta.LandReference,
+ LandReference: string(graftMeta.LandReference),
Lands: graftMeta.Lands,
}
if meta.Operation == "" {
diff --git a/internal/gitaly/service/operations/land.go b/internal/gitaly/service/operations/land.go
index 54f62df..cd8eccb 100644
--- a/internal/gitaly/service/operations/land.go
+++ b/internal/gitaly/service/operations/land.go
@@ -466,6 +466,13 @@ func validateUserLandRequest(ctx context.Context, locator storage.Locator, in *g
return fmt.Errorf("invalid branch name: %w", err)
}
+ // The land records its identity in its update (doc/graft/PLAN11.md), which bounds the name of
+ // the branch. The request is refused here rather than its update, so that it fails alone, also
+ // in a batch.
+ if err := changelog.CheckLandReference("refs/heads/" + string(in.GetBranch())); err != nil {
+ return err
+ }
+
if in.GetCommitId() == "" {
return errors.New("empty commit id")
}
diff --git a/internal/gitaly/service/operations/land_batch.go b/internal/gitaly/service/operations/land_batch.go
index 4897636..39bbfd9 100644
--- a/internal/gitaly/service/operations/land_batch.go
+++ b/internal/gitaly/service/operations/land_batch.go
@@ -323,6 +323,10 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequest
rewritten := false
var members []int
var changes []changelog.Member
+ // landedAs maps the commit of each member to the commit it landed as in the running tip, so
+ // that a later request of the same commit, a retry that joined the batch of its first land,
+ // is answered as already landed.
+ landedAs := map[git.ObjectID]git.ObjectID{}
for pos, i := range idx {
req := reqs[i]
outs[pos].OldTip = cur
@@ -349,6 +353,14 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequest
e.deliverEarly(ctx, i, len(reqs), outs[pos], final)
}
+ case step.tip == cur && landedAs[step.commit] != "":
+ // An earlier member of the batch landed the same commit: the request is a retry
+ // of it, and is answered as already landed with the commit of that member. The
+ // answer holds only if the update of the batch does: the request comes after a
+ // member, so when the update does not hold it is landed again (see run).
+ outs[pos].LandedCommit = landedAs[step.commit]
+ outs[pos].AlreadyLanded = true
+
case step.tip == cur:
// The commit became empty on top of the running tip and was dropped. The request
// reports a landing that changed nothing.
@@ -360,6 +372,7 @@ func (e landBatchExecutor) attempt(ctx context.Context, reqs []*landBatchRequest
members = append(members, pos)
changes = append(changes, memberOf(req, step))
+ landedAs[step.commit] = step.tip
outs[pos].LandedCommit = step.tip
outs[pos].Rewritten = step.rewritten
diff --git a/internal/gitaly/service/operations/land_identity.go b/internal/gitaly/service/operations/land_identity.go
index aecc04d..6b31413 100644
--- a/internal/gitaly/service/operations/land_identity.go
+++ b/internal/gitaly/service/operations/land_identity.go
@@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
+ "slices"
"strings"
"github.com/feder-cr/git-ai/internal/git"
@@ -29,9 +30,14 @@ type landIdentity struct {
revertedBy git.ObjectID
}
-// detail explains the answer, see UserLandResponse.already_landed_detail.
+// detail explains the answer, see UserLandResponse.already_landed_detail. A land known from its
+// intent has no changelog sequence: git committed it and the changelog did not record it, or has not
+// yet.
func (id *landIdentity) detail() string {
detail := fmt.Sprintf("landed as %s at changelog %d", id.landed, id.sequence)
+ if id.sequence == 0 {
+ detail = fmt.Sprintf("landed as %s, which the changelog does not record", id.landed)
+ }
if id.revertedBy != "" {
detail += fmt.Sprintf("; it was reverted later by %s", id.revertedBy)
}
@@ -60,44 +66,49 @@ func landWithRecord(ctx context.Context, reference git.ReferenceName, lands []ch
// landedEarlier returns the earlier land of commitID on the branch reference when it is still on
// the branch, that is when the commit it landed as is reachable from tip; nil otherwise. A change
// whose commit is not reachable from tip may have landed as a rewritten commit, and the record of
-// that land is the only way to tell (doc/graft/PLAN11.md, section 5). Only then is the commit that
-// reverted it looked for, to explain the answer.
+// that land is the only way to tell (doc/graft/PLAN11.md, section 5). The lookup gives the land the
+// changelog recorded first, then the intents of lands it does not record; the first on the branch
+// answers. An intent of an update that was not committed is not on the branch, and answers
+// nothing. Only once a land answers is the commit that reverted it looked for, to explain the
+// answer.
func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.ReferenceName, tip, commitID git.ObjectID) (*landIdentity, error) {
- record, found, err := a.server.lookupLand(ctx, a.repository, reference, commitID)
+ records, err := a.server.lookupLand(ctx, a.repository, reference, commitID)
if err != nil {
return nil, err
}
- if !found {
- return nil, nil
- }
- landed, err := a.objectHash.FromHex(record.Landed)
- if err != nil {
- return nil, structerr.NewInternal("land identity of %s: %w", commitID, err)
- }
+ for _, record := range records {
+ landed, err := a.objectHash.FromHex(record.Landed)
+ if err != nil {
+ return nil, structerr.NewInternal("land identity of %s: %w", commitID, err)
+ }
- onBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())
- if err != nil {
- // The branch was reset past the earlier land and its commit was pruned since: the change
- // is no longer on the branch.
- if errors.As(err, new(localrepo.InvalidCommitError)) {
- return nil, nil
+ onBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())
+ if err != nil {
+ // The commit does not exist: an intent of an update that was not committed, whose
+ // objects were never migrated, or a land whose branch was reset past it and whose
+ // commit was pruned since. Either way the change is not on the branch through it.
+ if errors.As(err, new(localrepo.InvalidCommitError)) {
+ continue
+ }
+ return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)
+ }
+ if !onBranch {
+ continue
}
- return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)
- }
- if !onBranch {
- return nil, nil
- }
- // The revert only explains the answer, so a failure to look for it does not change the
- // answer: it is logged, and the answer names no revert.
- revertedBy, err := a.revertOf(ctx, landed, tip)
- if err != nil {
- a.server.logger.WithError(err).WithField("landed", landed.String()).
- Error("graft: looking for the revert of an earlier land failed; the answer names no revert")
+ // The revert only explains the answer, so a failure to look for it does not change the
+ // answer: it is logged, and the answer names no revert.
+ revertedBy, err := a.revertOf(ctx, landed, tip)
+ if err != nil {
+ a.server.logger.WithError(err).WithField("landed", landed.String()).
+ Error("graft: looking for the revert of an earlier land failed; the answer names no revert")
+ }
+
+ return &landIdentity{landed: landed, sequence: record.Sequence, revertedBy: revertedBy}, nil
}
- return &landIdentity{landed: landed, sequence: record.Sequence, revertedBy: revertedBy}, nil
+ return nil, nil
}
// revertSearchCommits bounds the commits that revertOf reads: the revert is looked for among the
@@ -106,10 +117,10 @@ func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.Refere
var revertSearchCommits = 1000
// revertOf returns the latest commit after landed up to tip whose message holds "This reverts
-// commit <landed>", the line that git revert and graft-verify write, or an empty ID when there is
-// none among the revertSearchCommits commits closest to tip. A revert written another way, or
-// further from the tip, is not found. The revert only explains the answer: the land is already
-// landed whatever revertOf finds.
+// commit <landed>", the line that git revert and graft-verify write, and that no later commit
+// reverts in turn, or an empty ID when there is none among the revertSearchCommits commits closest
+// to tip. A revert written another way, or further from the tip, is not found. The revert only
+// explains the answer: the land is already landed whatever revertOf finds.
func (a *landingAttempt) revertOf(ctx context.Context, landed, tip git.ObjectID) (git.ObjectID, error) {
var stdout, stderr bytes.Buffer
if err := a.repo.ExecAndWait(ctx, gitcmd.Command{
@@ -124,59 +135,87 @@ func (a *landingAttempt) revertOf(ctx context.Context, landed, tip git.ObjectID)
}
// The output is pairs of fields, the commit ID and its message, each ended by a NUL, newest
- // commit first. Messages cannot hold a NUL.
- line := "This reverts commit " + landed.String()
+ // commit first. Messages cannot hold a NUL. The commits that a newer commit reverts are
+ // collected on the way, so that a revert that was reverted in turn is passed over.
+ revertedLater := map[string]bool{}
fields := strings.Split(stdout.String(), "\x00")
for i := 0; i+1 < len(fields); i += 2 {
- if !strings.Contains(fields[i+1], line) {
- continue
+ commit := strings.TrimSpace(fields[i])
+ reverts := revertedCommits(fields[i+1], a.objectHash.EncodedLen())
+
+ if slices.Contains(reverts, landed.String()) && !revertedLater[commit] {
+ revertedBy, err := a.objectHash.FromHex(commit)
+ if err != nil {
+ return "", fmt.Errorf("looking for a revert of %s: %w", landed, err)
+ }
+ return revertedBy, nil
}
- revertedBy, err := a.objectHash.FromHex(strings.TrimSpace(fields[i]))
- if err != nil {
- return "", fmt.Errorf("looking for a revert of %s: %w", landed, err)
+ for _, reverted := range reverts {
+ revertedLater[reverted] = true
}
- return revertedBy, nil
}
return "", nil
}
-// lookupLand returns the last land of original on the branch reference of repository that the
-// changelog recorded, from where the changelog lives: the storage core's database in its
-// authoritative mode, the file log of the repository otherwise. Without a changelog nothing is
-// recorded, and nothing is found.
-func (s *Server) lookupLand(ctx context.Context, repository *gitalypb.Repository, reference git.ReferenceName, original git.ObjectID) (changelog.LandRecord, bool, error) {
+// revertedCommits returns the commits that message says it reverts: the object IDs of length
+// hexLength that follow "This reverts commit ".
+func revertedCommits(message string, hexLength int) []string {
+ const prefix = "This reverts commit "
+
+ var reverted []string
+ for {
+ start := strings.Index(message, prefix)
+ if start < 0 {
+ return reverted
+ }
+ message = message[start+len(prefix):]
+ if len(message) >= hexLength {
+ reverted = append(reverted, message[:hexLength])
+ }
+ }
+}
+
+// lookupLand returns what is known of the lands of original on the branch reference of repository,
+// from where the changelog lives: in the storage core's authoritative mode, the land its database
+// recorded with the reference change, in one transaction; otherwise the land the file log recorded
+// and the land intents it does not record (see changelog.LookupLand). Without a changelog nothing
+// is recorded, and nothing is found.
+func (s *Server) lookupLand(ctx context.Context, repository *gitalypb.Repository, reference git.ReferenceName, original git.ObjectID) ([]changelog.LandRecord, error) {
if s.graftStorage.Authoritative() {
repo, err := s.graftStorage.Meta.GetRepository(ctx, corestore.KeyOf(repository))
if err != nil {
// A repository that the database does not track has no recorded land. Whether it
// can be written to is the write path's decision, as for any land.
if errors.Is(err, corestore.ErrRepositoryNotFound) {
- return changelog.LandRecord{}, false, nil
+ return nil, nil
}
- return changelog.LandRecord{}, false, structerr.NewUnavailable("storage core: %w", err)
+ return nil, structerr.NewUnavailable("storage core: %w", err)
}
record, found, err := s.graftStorage.Meta.LookupLand(ctx, repo.ID, reference.String(), original.String())
if err != nil {
- return changelog.LandRecord{}, false, structerr.NewUnavailable("storage core: %w", err)
+ return nil, structerr.NewUnavailable("storage core: %w", err)
}
- return record, found, nil
+ if !found {
+ return nil, nil
+ }
+ return []changelog.LandRecord{record}, nil
}
if s.graft.Changelog.Disabled {
- return changelog.LandRecord{}, false, nil
+ return nil, nil
}
repoPath, err := s.locator.GetRepoPath(ctx, repository)
if err != nil {
- return changelog.LandRecord{}, false, err
+ return nil, err
}
- record, found, err := changelog.LookupLand(repoPath, reference.String(), original.String())
+ records, err := changelog.LookupLand(repoPath, reference.String(), original.String())
if err != nil {
- return changelog.LandRecord{}, false, structerr.NewInternal("looking up the land identity: %w", err)
+ return nil, structerr.NewInternal("looking up the land identity: %w", err)
}
- return record, found, nil
+ return records, nil
}
diff --git a/internal/gitaly/service/operations/land_identity_test.go b/internal/gitaly/service/operations/land_identity_test.go
index 152b4ea..68039a5 100644
--- a/internal/gitaly/service/operations/land_identity_test.go
+++ b/internal/gitaly/service/operations/land_identity_test.go
@@ -2,13 +2,21 @@ package operations
import (
"context"
+ "errors"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
"testing"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/graft/changelog"
+ "github.com/feder-cr/git-ai/internal/testhelper"
)
// TestLandBatch_retryAfterALostResponse lands two changes by rebase in one batch, and then lands
@@ -73,6 +81,208 @@ func testLandBatchRetryAfterALostResponse(t *testing.T, ctx context.Context) {
requireBranchAt(t, cfg, repoPath, git.DefaultBranch, tip)
}
+// TestUserLand_intentAnswersWhatTheChangelogMisses lands a change whose update git committed and
+// whose changelog entry is lost, as a crash between the two leaves it (doc/graft/PLAN11.md, 4.2):
+// the intent the hook recorded before git committed answers the retry.
+func TestUserLand_intentAnswersWhatTheChangelogMisses(t *testing.T) {
+ t.Parallel()
+
+ runLandBatchTest(t, testUserLandIntentAnswersWhatTheChangelogMisses)
+}
+
+func testUserLandIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ ctx, cfg, client := setupOperationsService(t, ctx)
+ fx := newRetryFixture(t, ctx, cfg)
+
+ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)
+ first, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ landed := git.ObjectID(first.GetLandedCommitId())
+
+ // The changelog lost the entry of the land, its only entry.
+ require.NoError(t, os.Remove(changelog.Path(fx.repoPath)))
+
+ second, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ require.True(t, second.GetAlreadyLanded())
+ require.Equal(t, landed.String(), second.GetLandedCommitId())
+ require.Zero(t, second.GetLandedSequence())
+ require.Equal(t, fmt.Sprintf("landed as %s, which the changelog does not record", landed), second.GetAlreadyLandedDetail())
+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, landed)
+}
+
+// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing records intents whose updates git never
+// committed, as an update that was aborted after its prepared phase leaves them: their landed
+// commit is not on the branch, so the change lands.
+func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {
+ t.Parallel()
+
+ runLandBatchTest(t, testUserLandIntentOfAnUncommittedUpdateAnswersNothing)
+}
+
+func testUserLandIntentOfAnUncommittedUpdateAnswersNothing(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ ctx, cfg, client := setupOperationsService(t, ctx)
+ fx := newRetryFixture(t, ctx, cfg)
+
+ // One intent names a commit that does not exist, the other one that is not on the branch.
+ require.NoError(t, changelog.RecordLandIntents(fx.repoPath, git.DefaultRef.String(), []changelog.Land{
+ {Original: fx.change.String(), Landed: strings.Repeat("1", len(fx.change))},
+ {Original: fx.change.String(), Landed: fx.change.String()},
+ }))
+
+ resp, err := client.UserLand(ctx, landRequest(fx.repoProto, git.DefaultBranch, fx.change))
+ require.NoError(t, err)
+ require.False(t, resp.GetAlreadyLanded())
+ require.True(t, resp.GetRewritten())
+}
+
+// TestUserLand_identityOfABranchWhoseNameIsNotUTF8 lands a change by rebase on a branch whose name
+// is not valid UTF-8, through the hooks: its identity is recorded with the name byte for byte, and
+// the retry answers from it.
+func TestUserLand_identityOfABranchWhoseNameIsNotUTF8(t *testing.T) {
+ t.Parallel()
+
+ runLandBatchTest(t, testUserLandIdentityOfABranchWhoseNameIsNotUTF8)
+}
+
+func testUserLandIdentityOfABranchWhoseNameIsNotUTF8(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ ctx, cfg, client := setupOperationsService(t, ctx)
+ fx := newRetryFixture(t, ctx, cfg)
+ branch := "f\xffo"
+ gittest.Exec(t, cfg, "-C", fx.repoPath, "update-ref", "refs/heads/"+branch, fx.tip.String())
+
+ request := landRequest(fx.repoProto, branch, fx.change)
+ first, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ require.True(t, first.GetRewritten())
+ landed := git.ObjectID(first.GetLandedCommitId())
+
+ entries, err := changelog.List(fx.repoPath, 0, 100)
+ require.NoError(t, err)
+ require.Len(t, entries, 1)
+ require.Equal(t, "refs/heads/"+branch, entries[0].Reference)
+ require.Equal(t, []changelog.Land{{Original: fx.change.String(), Landed: landed.String()}}, entries[0].Lands)
+
+ second, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ requireLandedEarlier(t, second, entries[0], landed, "")
+}
+
+// TestUserLand_concurrentRetries lands the same change twice at once, as a retry that reaches the
+// server while the first call still runs: the change lands once, and the other call answers
+// already landed with the commit it landed as.
+func TestUserLand_concurrentRetries(t *testing.T) {
+ t.Parallel()
+
+ runLandBatchTest(t, testUserLandConcurrentRetries)
+}
+
+func testUserLandConcurrentRetries(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ graft config.Graft
+ }{
+ {desc: "unbatched", graft: config.Graft{}},
+ {desc: "batched", graft: agentBatchingGraft(nil)},
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ ctx, cfg, client := setupAgentLandService(t, ctx, tc.graft)
+ for round := 0; round < 5; round++ {
+ fx := newRetryFixture(t, ctx, cfg)
+ before := gittest.Exec(t, cfg, "-C", fx.repoPath, "rev-list", "--count", git.DefaultBranch)
+
+ responses, errs := landInOrder(ctx, client, fx.repoProto,
+ agentLanding{commit: fx.change},
+ agentLanding{commit: fx.change},
+ )
+ require.NoError(t, errs[0])
+ require.NoError(t, errs[1])
+
+ landed := 0
+ for _, resp := range responses {
+ if !resp.GetAlreadyLanded() {
+ landed++
+ require.True(t, resp.GetRewritten())
+ }
+ }
+ require.Equal(t, 1, landed, "round %d: the change lands once", round)
+ require.Equal(t, responses[0].GetLandedCommitId(), responses[1].GetLandedCommitId(), "round %d", round)
+
+ tip := gittest.ResolveRevision(t, cfg, fx.repoPath, git.DefaultRef.String())
+ require.Equal(t, responses[0].GetLandedCommitId(), tip.String(), "round %d", round)
+ require.Equal(t, mustAtoi(t, before)+1, mustAtoi(t, gittest.Exec(t, cfg, "-C", fx.repoPath, "rev-list", "--count", git.DefaultBranch)), "round %d", round)
+ }
+ })
+ }
+}
+
+// mustAtoi parses the decimal number of a git output.
+func mustAtoi(t *testing.T, output []byte) int {
+ t.Helper()
+
+ n, err := strconv.Atoi(strings.TrimSpace(string(output)))
+ require.NoError(t, err)
+ return n
+}
+
+// TestUserLand_branchNameTooLongForTheIdentity refuses, when it arrives, a land on a branch whose
+// name is too long for the identity of its update to be recorded.
+func TestUserLand_branchNameTooLongForTheIdentity(t *testing.T) {
+ t.Parallel()
+
+ ctx, cfg, client := setupOperationsService(t, testhelper.Context(t))
+ fx := newRetryFixture(t, ctx, cfg)
+
+ branch := strings.TrimSuffix(strings.Repeat(strings.Repeat("a", 200)+"/", 35), "/")
+ _, err := client.UserLand(ctx, landRequest(fx.repoProto, branch, fx.change))
+ testhelper.RequireGrpcCode(t, err, codes.InvalidArgument)
+ require.ErrorContains(t, err, "is too long to record the identity of a land on it")
+}
+
+// TestLandBatchExecutorRetryInTheBatchOfItsFirstLand puts a request and its retry in one batch:
+// the retry answers already landed with the commit of the first, and only when the batch's update
+// holds.
+func TestLandBatchExecutorRetryInTheBatchOfItsFirstLand(t *testing.T) {
+ t.Parallel()
+
+ t.Run("the update holds", func(t *testing.T) {
+ fake := &fakeLandStepper{tip: "main0"}
+ outcomes := newFakeLandExecutor(fake, 3).Execute(context.Background(), testLandRequests(false, "c1", "c1"))
+
+ require.NoError(t, outcomes[0].Err)
+ require.False(t, outcomes[0].AlreadyLanded)
+ require.NoError(t, outcomes[1].Err)
+ require.True(t, outcomes[1].AlreadyLanded)
+ require.Equal(t, git.ObjectID("c1"), outcomes[1].LandedCommit)
+ require.Len(t, fake.updates, 1)
+ require.Equal(t, []string{"c1"}, fake.updates[0].members2Commits(), "the retry is not a member")
+ })
+
+ t.Run("the update fails", func(t *testing.T) {
+ // The first land fails with the update. Its retry is not answered as landed: it lands
+ // again, alone, onto the real tip.
+ fake := &fakeLandStepper{tip: "main0", script: []scriptedUpdate{{kind: landUpdateFailed, err: errors.New("disk full")}}}
+ outcomes := newFakeLandExecutor(fake, 3).Execute(context.Background(), testLandRequests(false, "c1", "c1"))
+
+ require.ErrorContains(t, outcomes[0].Err, "disk full")
+ require.NoError(t, outcomes[1].Err)
+ require.False(t, outcomes[1].AlreadyLanded)
+ require.Equal(t, git.ObjectID("c1"), outcomes[1].LandedCommit)
+ require.Len(t, fake.updates, 2)
+ require.Equal(t, []string{"c1"}, fake.updates[1].members2Commits())
+ })
+}
+
// TestLandBatch_retryOfABatchOfOne retries a change that landed alone in its lane, which publishes
// it as a land of its own (UpdateSolo): its identity is recorded, and the retry answers from it.
func TestLandBatch_retryOfABatchOfOne(t *testing.T) {
diff --git a/internal/gitaly/service/operations/land_retry_test.go b/internal/gitaly/service/operations/land_retry_test.go
index bc4e456..b7c330f 100644
--- a/internal/gitaly/service/operations/land_retry_test.go
+++ b/internal/gitaly/service/operations/land_retry_test.go
@@ -238,6 +238,35 @@ func testUserLandRetryAfterALostResponse(t *testing.T, ctx context.Context) {
}
})
+ t.Run("after a revert that was reverted in turn", func(t *testing.T) {
+ t.Parallel()
+
+ // The change landed, a commit reverted it, and a later commit reverted the revert: the
+ // change is back on the branch, and the answer names no revert.
+ fx := newRetryFixture(t, ctx, cfg)
+ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)
+ first, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ landed := git.ObjectID(first.GetLandedCommitId())
+
+ reverted := gittest.WriteCommit(t, cfg, fx.repoPath,
+ gittest.WithBranch(git.DefaultBranch),
+ gittest.WithParents(landed),
+ gittest.WithTreeEntries(fx.withoutY...),
+ gittest.WithMessage(fmt.Sprintf("Revert \"y\"\n\nThis reverts commit %s.\n", landed)),
+ )
+ gittest.WriteCommit(t, cfg, fx.repoPath,
+ gittest.WithBranch(git.DefaultBranch),
+ gittest.WithParents(reverted),
+ gittest.WithTreeEntries(append(fx.withoutY, gittest.TreeEntry{Mode: "100644", Path: "y", Content: "y"})...),
+ gittest.WithMessage(fmt.Sprintf("Revert \"Revert \"y\"\"\n\nThis reverts commit %s.\n", reverted)),
+ )
+
+ second, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ requireLandedEarlier(t, second, landEntry(t, fx.repoPath, landed), landed, "")
+ })
+
t.Run("after the branch was reset past the land", func(t *testing.T) {
t.Parallel()
diff --git a/internal/graft/changelog/lands.go b/internal/graft/changelog/lands.go
index 973b52c..c677397 100644
--- a/internal/graft/changelog/lands.go
+++ b/internal/graft/changelog/lands.go
@@ -2,6 +2,7 @@ package changelog
import (
"context"
+ "encoding/base64"
"encoding/json"
"fmt"
"strings"
@@ -32,22 +33,49 @@ const (
// MaxLandsSize bounds the encoded size of the identities of one update (the reference and its
// lands), which travel in the hooks payload, in the environment of every git process of the
// update, next to the members (see gitcmd.ChangelogMeta). Unlike the members, the identities are
- // never cut: an update whose identities do not fit fails before it runs (WithLands). With the
- // limits of the server it cannot happen: a batch has at most MaxLandsPerUpdate members, a land
- // of a SHA-256 repository encodes in 156 bytes, and a branch name of 4096 bytes, which no files
- // reference backend exceeds, encodes in at most 6 bytes per byte: 40 KiB hold both with room.
- MaxLandsSize = 40 << 10
+ // never cut: a land request whose branch name could not fit is refused when it arrives
+ // (CheckLandReference), and WithLands checks the bound again before the update runs. A batch
+ // has at most MaxLandsPerUpdate members and a land of a SHA-256 repository encodes in 156
+ // bytes, 15.6 KB for a full batch; the branch name is encoded in base64, so 24 KiB hold a full
+ // batch and a branch name of 4096 bytes, which no files reference backend exceeds, with room.
+ MaxLandsSize = 24 << 10
// MaxLandsPerUpdate is the most lands that one update records: the largest batch the
// configuration allows (config.MaxGraftLandBatchMaxSize).
MaxLandsPerUpdate = 100
)
-// encodedLands is the encoded form of the identities of one update whose size MaxLandsSize bounds.
+// encodedLands is the encoded form of the identities of one update whose size MaxLandsSize bounds,
+// as gitcmd.ChangelogMeta encodes them: the reference as bytes, in base64.
type encodedLands struct {
- Reference string `json:"land_reference"`
+ Reference []byte `json:"land_reference"`
Lands []Land `json:"lands"`
}
+// fullBatchLandsSize is the encoded size of the identities of a full batch of SHA-256 lands on a
+// branch with an empty name, the largest identities of any branch but its name.
+var fullBatchLandsSize = func() int {
+ lands := make([]Land, MaxLandsPerUpdate)
+ for i := range lands {
+ lands[i] = Land{Original: strings.Repeat("f", 64), Landed: strings.Repeat("f", 64)}
+ }
+ encoded, err := json.Marshal(encodedLands{Lands: lands})
+ if err != nil {
+ panic(fmt.Sprintf("encoding the identities of a full batch: %v", err))
+ }
+ return len(encoded)
+}()
+
+// CheckLandReference checks, when a land request arrives, that the identities of any update of
+// reference fit MaxLandsSize: those of a full batch of SHA-256 lands on it. A branch whose name is
+// too long for them cannot be landed on, since a land would not record its identity.
+func CheckLandReference(reference string) error {
+ if size := fullBatchLandsSize + base64.StdEncoding.EncodedLen(len(reference)); size > MaxLandsSize {
+ return fmt.Errorf("the name of branch %q is too long to record the identity of a land on it: %d bytes, at most %d",
+ reference, len(reference), (MaxLandsSize-fullBatchLandsSize)/4*3)
+ }
+ return nil
+}
+
// ValidateLands checks the identities that an update of reference records: reference is a branch,
// every land names two full object IDs, there are at most MaxLandsPerUpdate of them and their
// encoding fits MaxLandsSize.
@@ -68,7 +96,7 @@ func ValidateLands(reference string, lands []Land) error {
}
}
- encoded, err := json.Marshal(encodedLands{Reference: reference, Lands: lands})
+ encoded, err := json.Marshal(encodedLands{Reference: []byte(reference), Lands: lands})
if err != nil {
return fmt.Errorf("land identities of %q: %w", reference, err)
}
diff --git a/internal/graft/changelog/lands_index.go b/internal/graft/changelog/lands_index.go
index aae0e68..222e12a 100644
--- a/internal/graft/changelog/lands_index.go
+++ b/internal/graft/changelog/lands_index.go
@@ -10,23 +10,35 @@ import (
"io"
"os"
"path/filepath"
+ "sort"
"sync"
+ "sync/atomic"
)
const (
// landsFileName is the name of the index of the land identities inside dirName.
landsFileName = "changelog.lands"
// maxIdleLandsIndexes bounds the number of indexes that are kept in memory while no lookup
- // uses them. An index holds every land of its repository, so fewer of them are kept than of
- // the states of the logs (maxIdleStates). A dropped index is loaded from its file again.
+ // uses them. A dropped index is loaded from its file again.
maxIdleLandsIndexes = 64
+ // maxIdleLandsBytes bounds the memory of the indexes that are kept while no lookup uses them,
+ // as estimated by landsIndex.bytes. The least recently used ones beyond it are dropped, so an
+ // index larger than the bound is dropped as soon as its lookup ends.
+ maxIdleLandsBytes = 256 << 20
+ // compactLandsMinSize is the size from which an index file is compacted once most of its
+ // records are superseded by later ones.
+ compactLandsMinSize = 1 << 20
+ // compactIntentsMinSize is the size from which an intents file is compacted, see
+ // compactIntents.
+ compactIntentsMinSize = 64 << 10
)
// The index of the land identities of a repository (doc/graft/PLAN11.md, 4.2) maps a branch and an
// original commit to the last changelog entry that landed that commit on that branch. It is derived
// from the changelog: the log entries carry the lands (Entry.Lands), and the index is the result of
// reading them, kept in memory and in the file graft/changelog.lands so that a new process does not
-// read the whole log again.
+// read the whole log again. A lookup also reads the land intents (see landIntent), which cover the
+// lands whose update git committed before the changelog recorded it.
//
// The file is a sequence of blocks of JSON lines. A block holds the records of the lands of some
// log entries, in log order, and ends with a mark: the sequence of the last log entry the block
@@ -41,6 +53,10 @@ const (
// entry is not the one the mark names (the log was replaced, for example by a repository created
// again at the same path) the index is rebuilt from the whole log. The file is therefore written
// without fsync: what a crash leaves of it is either a good prefix or is cut off.
+//
+// In memory, the index also knows the log file it was built from: a log file that is another file
+// (another inode) rebuilds it, and a lookup whose log ends with the entry of the mark, in the same
+// file, reads nothing.
// landsIndexKey identifies a land in an index.
type landsIndexKey struct {
@@ -60,29 +76,55 @@ func (m landsMark) matches(entry Entry) bool {
return entry.Sequence == m.sequence && entry.Reference == m.reference && entry.NewOID == m.newOID
}
+// describes reports whether head is the head of the log that the mark names the last entry of.
+func (m landsMark) describes(head logHead) bool {
+ return head.sequence == m.sequence && head.reference == m.reference && head.newOID == m.newOID
+}
+
// landsIndex is the in-process view of the index of one repository.
type landsIndex struct {
// mu serializes the lookups of this process in the repository and guards the fields below.
mu sync.Mutex
- // users counts the callers that hold the index. It is guarded by landsIndexesMu.
- users int
+ // users and lastUsed are guarded by landsIndexesMu: the callers that hold the index, and the
+ // tick of its last release, which orders the eviction.
+ users int
+ lastUsed uint64
+ // bytes is the estimate of the memory of the index, see estimate. It is read by the eviction
+ // without mu.
+ bytes atomic.Int64
+
// loaded is false until the index has been read from its file.
loaded bool
// mark is the log entry up to which records is complete; its sequence is zero when records
// covers no entry.
mark landsMark
+ // logDevice and logInode identify the log file that records was read from.
+ logDevice, logInode uint64
// records maps the lands of the log entries up to mark to their last record.
records map[landsIndexKey]LandRecord
+ // recordLines is the number of record lines of the index file, superseded ones included.
+ recordLines int
// fileSize is the size of the index file when this process last read or wrote it. A file of
// another size was written by another process and is read again.
fileSize int64
+
+ // intents are the land intents read from the intents file, by land, and intentsFile what is
+ // known of that file.
+ intents map[landsIndexKey][]landIntent
+ intentsFile intentsFile
+ // intentsKept is the size of the intents file after its last compaction by this process. The
+ // file is compacted again once it doubled, so that intents that cannot go yet are not
+ // rewritten by every lookup.
+ intentsKept int64
}
var (
landsIndexesMu sync.Mutex
// landsIndexes holds the indexes by the path of their file. An index is kept after its last
- // user releases it, up to maxIdleLandsIndexes idle ones.
+ // user releases it, within maxIdleLandsIndexes and maxIdleLandsBytes.
landsIndexes = map[string]*landsIndex{}
+ // landsTick orders the releases of the indexes. It is guarded by landsIndexesMu.
+ landsTick uint64
)
// acquireLandsIndex returns the index whose file is path, and the function that releases it.
@@ -101,32 +143,45 @@ func acquireLandsIndex(path string) (*landsIndex, func()) {
defer landsIndexesMu.Unlock()
index.users--
+ landsTick++
+ index.lastUsed = landsTick
evictIdleLandsIndexes()
}
}
-// evictIdleLandsIndexes drops idle indexes until at most maxIdleLandsIndexes remain idle. The caller
-// must hold landsIndexesMu.
+// evictIdleLandsIndexes drops the least recently used idle indexes until at most
+// maxIdleLandsIndexes of them, of at most maxIdleLandsBytes together, remain. The caller must hold
+// landsIndexesMu.
func evictIdleLandsIndexes() {
- if len(landsIndexes) <= maxIdleLandsIndexes {
- return
+ type idleIndex struct {
+ path string
+ lastUsed uint64
+ bytes int64
}
- idle := 0
- for _, index := range landsIndexes {
+ var (
+ idle []idleIndex
+ total int64
+ )
+ for path, index := range landsIndexes {
if index.users == 0 {
- idle++
+ bytes := index.bytes.Load()
+ idle = append(idle, idleIndex{path: path, lastUsed: index.lastUsed, bytes: bytes})
+ total += bytes
}
}
+ if len(idle) <= maxIdleLandsIndexes && total <= maxIdleLandsBytes {
+ return
+ }
- for path, index := range landsIndexes {
- if idle <= maxIdleLandsIndexes {
+ sort.Slice(idle, func(a, b int) bool { return idle[a].lastUsed < idle[b].lastUsed })
+ for _, index := range idle {
+ if len(idle) <= maxIdleLandsIndexes && total <= maxIdleLandsBytes {
return
}
- if index.users == 0 {
- delete(landsIndexes, path)
- idle--
- }
+ delete(landsIndexes, index.path)
+ idle = idle[1:]
+ total -= index.bytes
}
}
@@ -136,12 +191,14 @@ func landsPath(repoPath string) string {
return filepath.Join(repoPath, dirName, landsFileName)
}
-// LookupLand returns the last land of original on reference that the changelog of the repository
-// whose git directory is repoPath recorded (doc/graft/PLAN11.md): the sequence of its entry and the
-// commit it landed as. found is false when the changelog recorded no land of original on reference.
-// The lookup reads the durable entries of the log (see AppendGroup) that the index does not cover
-// yet, and adds them to the index first.
-func LookupLand(repoPath, reference, original string) (record LandRecord, found bool, err error) {
+// LookupLand returns what the repository whose git directory is repoPath knows of the lands of
+// original on reference (doc/graft/PLAN11.md): first the last land that the changelog recorded,
+// with the sequence of its entry and the commit it landed as, then the land intents (see
+// landIntent) of lands that the changelog does not record, with a zero sequence. A land answers
+// only when its landed commit is on the branch, which the caller checks. The result is empty when
+// nothing is known. The lookup reads the durable entries of the log (see AppendGroup) and the
+// intents that the index does not cover yet, and adds them to the index first.
+func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {
index, release := acquireLandsIndex(landsPath(repoPath))
defer release()
@@ -151,26 +208,71 @@ func LookupLand(repoPath, reference, original string) (record LandRecord, found
if err := index.catchUp(repoPath); err != nil {
// What the failed catch-up did to the index is not known, so the next lookup reads it again.
index.loaded = false
- return LandRecord{}, false, err
+ index.intentsFile = intentsFile{}
+ return nil, err
+ }
+
+ key := landsIndexKey{reference: reference, original: original}
+ var lands []LandRecord
+ confirmed, found := index.records[key]
+ if found {
+ lands = append(lands, confirmed)
+ }
+ for _, intent := range index.intents[key] {
+ if found && (intent.Landed == confirmed.Landed || confirmed.Sequence > intent.Head) {
+ continue
+ }
+ lands = append(lands, LandRecord{Landed: intent.Landed})
}
- record, found = index.records[landsIndexKey{reference: reference, original: original}]
- return record, found, nil
+ return lands, nil
}
-// catchUp brings the index up to the last durable entry of the log. The caller must hold i.mu.
+// catchUp brings the index up to the last durable entry of the log and the intents up to the end
+// of their file, and compacts their files when they grew large. The caller must hold i.mu.
func (i *landsIndex) catchUp(repoPath string) error {
- last, err := LastSequence(repoPath)
+ if err := i.catchUpLog(repoPath); err != nil {
+ return err
+ }
+ if err := i.catchUpIntents(repoPath); err != nil {
+ return err
+ }
+ i.bytes.Store(i.estimate())
+
+ return nil
+}
+
+// estimate returns an estimate of the memory of the index: its strings and the overhead of maps
+// and slices. The caller must hold i.mu.
+func (i *landsIndex) estimate() int64 {
+ const overhead = 96
+ var bytes int64
+ for key, record := range i.records {
+ bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)) + overhead
+ }
+ for key, intents := range i.intents {
+ for _, intent := range intents {
+ bytes += int64(len(key.reference)+len(key.original)+len(intent.Landed)+len(intent.Reference)) + overhead
+ }
+ }
+ return bytes
+}
+
+// catchUpLog brings the index up to the last durable entry of the log. The caller must hold i.mu.
+func (i *landsIndex) catchUpLog(repoPath string) error {
+ head, err := readLogHead(repoPath)
if err != nil {
return err
}
- if last == 0 {
+ if head.sequence == 0 {
// A log without entries recorded no land. The index, in memory or in its file, is of
// another log if it holds anything, and is rebuilt once this log has entries.
i.reset(0)
return nil
}
- if i.loaded && last == i.mark.sequence {
+
+ sameLog := i.loaded && head.device == i.logDevice && head.inode == i.logInode
+ if sameLog && i.mark.describes(head) {
return nil
}
@@ -195,13 +297,20 @@ func (i *landsIndex) catchUp(repoPath string) error {
if err != nil {
return fmt.Errorf("stat land index: %w", err)
}
- if !i.loaded || info.Size() != i.fileSize {
+
+ block, err := landsBlock{}, errLandsIndexStale
+ switch {
+ case i.loaded && !sameLog:
+ // The log is another file than the one the index was read from: the repository was
+ // created again. The index is rebuilt.
+ case !i.loaded || info.Size() != i.fileSize:
if err := i.load(f); err != nil {
return err
}
+ block, err = i.scanLog(repoPath)
+ default:
+ block, err = i.scanLog(repoPath)
}
-
- block, err := i.scanLog(repoPath)
if errors.Is(err, errLandsIndexStale) {
// The log is not the one the index was built from: build it again from the whole log.
if err := truncateFile(f, 0); err != nil {
@@ -214,25 +323,109 @@ func (i *landsIndex) catchUp(repoPath string) error {
return err
}
- if block.mark.sequence == i.mark.sequence {
- i.loaded = true
- return nil
+ if block.mark.sequence != i.mark.sequence {
+ encoded, err := block.encode()
+ if err != nil {
+ return err
+ }
+ if _, err := f.WriteAt(encoded, i.fileSize); err != nil {
+ return fmt.Errorf("writing land index: %w", err)
+ }
+
+ for _, record := range block.records {
+ i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}
+ }
+ i.recordLines += len(block.records)
+ i.mark = block.mark
+ i.fileSize += int64(len(encoded))
}
+ i.logDevice, i.logInode = head.device, head.inode
+ i.loaded = true
+
+ if i.fileSize >= compactLandsMinSize && i.recordLines > 2*len(i.records) {
+ return i.compact(path)
+ }
+
+ return nil
+}
+
+// compact rewrites the index file with only the records that are not superseded, as one block,
+// and replaces the old file with it. The caller must hold i.mu and the lock file of the index; the
+// rename makes the other processes read the new file.
+func (i *landsIndex) compact(path string) error {
+ block := landsBlock{mark: i.mark}
+ for key, record := range i.records {
+ block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed})
+ }
+ sort.Slice(block.records, func(a, b int) bool { return block.records[a].sequence < block.records[b].sequence })
+
encoded, err := block.encode()
if err != nil {
return err
}
- if _, err := f.WriteAt(encoded, i.fileSize); err != nil {
- return fmt.Errorf("writing land index: %w", err)
+
+ tmp := path + ".compact"
+ if err := os.WriteFile(tmp, encoded, 0o644); err != nil {
+ return fmt.Errorf("compacting land index: %w", err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ return fmt.Errorf("compacting land index: %w", err)
}
- for _, record := range block.records {
- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}
+ i.recordLines = len(block.records)
+ i.fileSize = int64(len(encoded))
+
+ return nil
+}
+
+// catchUpIntents reads the land intents recorded since the last lookup, and compacts the intents
+// file when it grew large. The caller must hold i.mu.
+func (i *landsIndex) catchUpIntents(repoPath string) error {
+ path := intentsPath(repoPath)
+
+ add := func(intent landIntent) {
+ key := landsIndexKey{reference: string(intent.Reference), original: intent.Original}
+ i.intents[key] = append(i.intents[key], intent)
}
- i.mark = block.mark
- i.fileSize += int64(len(encoded))
- i.loaded = true
+
+ if i.intents == nil {
+ i.intents = map[landsIndexKey][]landIntent{}
+ }
+ known, reset, err := readIntents(path, i.intentsFile, add)
+ if err != nil {
+ return err
+ }
+ if reset {
+ // The file is not the one that was read before, and the intents read before are not
+ // its: they are dropped, and the file is read again from its start.
+ i.intents = map[landsIndexKey][]landIntent{}
+ if known, _, err = readIntents(path, intentsFile{}, add); err != nil {
+ return err
+ }
+ }
+ i.intentsFile = known
+
+ if known.offset < compactIntentsMinSize || known.offset < 2*i.intentsKept {
+ return nil
+ }
+
+ // An intent may go once the changelog confirms it (a record of its land with its landed
+ // commit) or supersedes it (a later record of the same original commit on the branch).
+ if err := compactIntents(repoPath, path, func(intent landIntent) bool {
+ record, found := i.records[landsIndexKey{reference: string(intent.Reference), original: intent.Original}]
+ return found && (record.Landed == intent.Landed || record.Sequence > intent.Head)
+ }); err != nil {
+ return err
+ }
+
+ i.intents = map[landsIndexKey][]landIntent{}
+ known, _, err = readIntents(path, intentsFile{}, add)
+ if err != nil {
+ return err
+ }
+ i.intentsFile = known
+ i.intentsKept = known.offset
return nil
}
@@ -350,6 +543,7 @@ func (b landsBlock) encode() ([]byte, error) {
func (i *landsIndex) reset(fileSize int64) {
i.mark = landsMark{}
i.records = map[landsIndexKey]LandRecord{}
+ i.recordLines = 0
i.fileSize = fileSize
i.loaded = false
}
@@ -408,6 +602,7 @@ func (i *landsIndex) load(f *os.File) error {
for _, record := range pending {
i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}
}
+ i.recordLines += len(pending)
i.mark = landsMark{sequence: decoded.Mark, reference: string(decoded.Reference), newOID: decoded.NewOID}
i.fileSize = offset
pending = nil
diff --git a/internal/graft/changelog/lands_intents.go b/internal/graft/changelog/lands_intents.go
new file mode 100644
index 0000000..772b8f8
--- /dev/null
+++ b/internal/graft/changelog/lands_intents.go
@@ -0,0 +1,259 @@
+package changelog
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sync"
+ "syscall"
+)
+
+// intentsFileName is the name of the file of the land intents inside dirName.
+const intentsFileName = "changelog.intents"
+
+// The land intents of a repository (doc/graft/PLAN11.md, 4.2) close the window between git
+// committing the update of a land and the changelog recording it. The changelog entry of an update,
+// which carries its lands, is written in the committed phase of the reference transaction, after
+// git committed the references: a crash, a full disk or a lock that times out in between leaves the
+// branch with the land and the changelog without its identity. So, in the prepared phase, before
+// git may commit, the hook records the lands of the update as intents and fsyncs them. A lookup
+// answers from the confirmed records of the index and from the intents; the land path counts an
+// intent only when its landed commit is on the branch, so the intent of an update that was not
+// committed answers nothing.
+//
+// The file is JSON lines, appended under the lock file of the intents and fsync'd before the
+// prepared phase ends. Unlike the index it is not derived data. An intent is dropped by a
+// compaction once the changelog confirms it (a record of its land with its landed commit) or
+// supersedes it (a later record of the same original commit on the branch); the intent of an
+// update that was never committed and never landed again stays.
+
+// landIntent is one line of the intents file.
+type landIntent struct {
+ // Reference is the branch, as bytes (base64 in JSON): branch names may hold bytes that are
+ // not valid UTF-8.
+ Reference []byte `json:"ref"`
+ Original string `json:"original"`
+ Landed string `json:"landed"`
+ // Head is the sequence of the last durable changelog entry when the intent was recorded. The
+ // entry of the land, if any, comes after it, so a record of the same commit at a later
+ // sequence confirms or supersedes the intent.
+ Head uint64 `json:"head"`
+}
+
+// intentsPath returns the path of the intents file of the repository whose git directory is
+// repoPath.
+func intentsPath(repoPath string) string {
+ return filepath.Join(repoPath, dirName, intentsFileName)
+}
+
+// intentDirsSynced holds the intents files of this process whose directory entries are durable,
+// see RecordLandIntents.
+var intentDirsSynced sync.Map
+
+// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the
+// update of reference about to be committed lands lands (see the comment above landIntent). It
+// returns once the intents are fsync'd; an error must abort the update, which would otherwise be
+// committed without its identity.
+func RecordLandIntents(repoPath, reference string, lands []Land) error {
+ head, err := readLogHead(repoPath)
+ if err != nil {
+ return err
+ }
+
+ path := intentsPath(repoPath)
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ return fmt.Errorf("creating changelog directory: %w", err)
+ }
+
+ unlock, err := lockFile(path + lockSuffix)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
+ f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
+ if err != nil {
+ return fmt.Errorf("opening land intents: %w", err)
+ }
+ defer f.Close()
+
+ // The bytes after the last newline are a torn line of a writer that crashed before its fsync,
+ // whose update was therefore never committed. They are cut off, or the next line would be
+ // joined to them.
+ end, err := endOfLastLine(f)
+ if err != nil {
+ return err
+ }
+
+ var buf bytes.Buffer
+ for _, land := range lands {
+ line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Head: head.sequence})
+ if err != nil {
+ return fmt.Errorf("encoding land intent: %w", err)
+ }
+ buf.Write(line)
+ buf.WriteByte('\n')
+ }
+
+ if err := truncateFile(f, end); err != nil {
+ return fmt.Errorf("truncating land intents: %w", err)
+ }
+ if _, err := f.WriteAt(buf.Bytes(), end); err != nil {
+ return fmt.Errorf("writing land intents: %w", err)
+ }
+ if err := syncFile(f); err != nil {
+ return fmt.Errorf("syncing land intents: %w", err)
+ }
+
+ // The intents are durable only once the directory entries of their file are, which a writer
+ // that created the file may have crashed before syncing: the first write of this process
+ // syncs them.
+ if _, synced := intentDirsSynced.Load(path); !synced {
+ if err := syncLogDirs(repoPath, path); err != nil {
+ return err
+ }
+ intentDirsSynced.Store(path, struct{}{})
+ }
+
+ return nil
+}
+
+// endOfLastLine returns the offset just after the last newline of f, 0 when it has none.
+func endOfLastLine(f *os.File) (int64, error) {
+ info, err := f.Stat()
+ if err != nil {
+ return 0, fmt.Errorf("stat land intents: %w", err)
+ }
+
+ nl, err := lastNewlineBefore(f, info.Size())
+ if err != nil {
+ return 0, err
+ }
+
+ return nl + 1, nil
+}
+
+// intentsFile is what a reader of an intents file knows of it.
+type intentsFile struct {
+ // device and inode identify the file. A compaction replaces it.
+ device, inode uint64
+ // offset is the end of the complete lines read so far.
+ offset int64
+}
+
+// readIntents reads the complete lines of the intents file at path after known, calling add for
+// each intent. It returns what the reader knows of the file after the read. When the file is not
+// the one known (it was replaced) or is shorter (it was replaced or cut), it is read from the
+// start and reset is true: the intents read before do not describe it any more. A missing file has
+// no intents.
+func readIntents(path string, known intentsFile, add func(landIntent)) (intentsFile, bool, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return intentsFile{}, known != (intentsFile{}), nil
+ }
+ return known, false, fmt.Errorf("opening land intents: %w", err)
+ }
+ defer f.Close()
+
+ info, err := f.Stat()
+ if err != nil {
+ return known, false, fmt.Errorf("stat land intents: %w", err)
+ }
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return known, false, fmt.Errorf("stat land intents: no file identity in %T", info.Sys())
+ }
+
+ current := intentsFile{device: uint64(stat.Dev), inode: stat.Ino, offset: known.offset}
+ reset := current.device != known.device || current.inode != known.inode || info.Size() < known.offset
+ if reset {
+ current.offset = 0
+ }
+
+ r := bufio.NewReader(io.NewSectionReader(f, current.offset, info.Size()-current.offset))
+ for {
+ line, err := r.ReadBytes('\n')
+ if err != nil && !errors.Is(err, io.EOF) {
+ return known, false, fmt.Errorf("reading land intents: %w", err)
+ }
+ // A line without its newline is still being written, or is the torn line of a writer that
+ // crashed: it is read once it is complete, or cut off by the next writer.
+ if len(line) == 0 || line[len(line)-1] != '\n' {
+ break
+ }
+ current.offset += int64(len(line))
+
+ // A line that does not decode is not an intent. Only a writer that crashed before its
+ // fsync leaves one, and its update was not committed.
+ var intent landIntent
+ if json.Unmarshal(line, &intent) != nil || !isObjectID(intent.Original) || !isObjectID(intent.Landed) {
+ continue
+ }
+ add(intent)
+ }
+
+ return current, reset, nil
+}
+
+// compactIntents rewrites the intents file at path without the intents that drop says may go,
+// under the lock file of the intents, so that no intent recorded meanwhile is lost: the file is
+// read whole under the lock. The new file is fsync'd and renamed over the old one, and the
+// directory is fsync'd, before the lock is released.
+func compactIntents(repoPath, path string, drop func(landIntent) bool) error {
+ unlock, err := lockFile(path + lockSuffix)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
+ var (
+ kept bytes.Buffer
+ encodeErr error
+ )
+ if _, _, err := readIntents(path, intentsFile{}, func(intent landIntent) {
+ if drop(intent) || encodeErr != nil {
+ return
+ }
+ line, err := json.Marshal(intent)
+ if err != nil {
+ encodeErr = fmt.Errorf("encoding land intent: %w", err)
+ return
+ }
+ kept.Write(line)
+ kept.WriteByte('\n')
+ }); err != nil {
+ return err
+ }
+ if encodeErr != nil {
+ return encodeErr
+ }
+
+ tmp := path + ".compact"
+ f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
+ if err != nil {
+ return fmt.Errorf("compacting land intents: %w", err)
+ }
+ if _, err := f.Write(kept.Bytes()); err != nil {
+ f.Close()
+ return fmt.Errorf("compacting land intents: %w", err)
+ }
+ if err := syncFile(f); err != nil {
+ f.Close()
+ return fmt.Errorf("compacting land intents: %w", err)
+ }
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("compacting land intents: %w", err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ return fmt.Errorf("compacting land intents: %w", err)
+ }
+
+ return syncLogDirs(repoPath, path)
+}
diff --git a/internal/graft/changelog/lands_intents_test.go b/internal/graft/changelog/lands_intents_test.go
new file mode 100644
index 0000000..f42268a
--- /dev/null
+++ b/internal/graft/changelog/lands_intents_test.go
@@ -0,0 +1,325 @@
+package changelog
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// recordIntent records the intent of a land of original on reference as landed.
+func recordIntent(t *testing.T, repoPath, reference, original, landed string) {
+ t.Helper()
+ require.NoError(t, RecordLandIntents(repoPath, reference, []Land{{Original: original, Landed: landed}}))
+}
+
+// intentsOf returns the intents in the intents file of the repository.
+func intentsOf(t *testing.T, repoPath string) []landIntent {
+ t.Helper()
+
+ var intents []landIntent
+ _, _, err := readIntents(intentsPath(repoPath), intentsFile{}, func(intent landIntent) {
+ intents = append(intents, intent)
+ })
+ require.NoError(t, err)
+ return intents
+}
+
+func TestLookupLandAnswersFromIntents(t *testing.T) {
+ t.Parallel()
+
+ t.Run("an intent without a log", func(t *testing.T) {
+ // Git committed the first land of the repository, and the process stopped before the
+ // changelog recorded it.
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+
+ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/other", landOID(1)))
+ })
+
+ t.Run("an intent that the changelog confirms", func(t *testing.T) {
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+
+ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ })
+
+ t.Run("an intent that a later land supersedes", func(t *testing.T) {
+ // The update of the intent was not committed, and the change was landed again.
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(3))
+
+ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(3)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ })
+
+ t.Run("an intent after the recorded land", func(t *testing.T) {
+ // The change landed, the branch was reset past it, and its new land was committed
+ // without the changelog recording it: both are known, the record first.
+ repoPath := t.TempDir()
+ seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))
+
+ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}, {Landed: landOID(3)}},
+ lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ })
+
+ t.Run("a branch whose name is not valid UTF-8", func(t *testing.T) {
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/\xff", landOID(1), landOID(2))
+
+ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/\xff", landOID(1)))
+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/�", landOID(1)))
+ })
+
+ t.Run("intents recorded by another process", func(t *testing.T) {
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ require.Len(t, lookupAll(t, repoPath, "refs/heads/main", landOID(1)), 1)
+
+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))
+ require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))
+ })
+}
+
+func TestLandIntentsFile(t *testing.T) {
+ t.Parallel()
+
+ t.Run("a torn line is cut off by the next intent", func(t *testing.T) {
+ // A writer crashed in the middle of its line, before its fsync: its update was not
+ // committed. The next writer cuts the line off, so that its own line stays whole.
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ appendToFile(t, intentsPath(repoPath), `{"ref":"cmVmcy9oZWFkcy9tYWlu","original":"`+landOID(5))
+
+ _, err := LookupLand(repoPath, "refs/heads/main", landOID(5))
+ require.NoError(t, err, "a reader ignores the torn line")
+
+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))
+ require.Len(t, intentsOf(t, repoPath), 2)
+ require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))
+ require.Empty(t, lookupAll(t, repoPath, "refs/heads/main", landOID(5)))
+ })
+
+ t.Run("a line that is not an intent is skipped", func(t *testing.T) {
+ repoPath := t.TempDir()
+ require.NoError(t, os.MkdirAll(filepath.Dir(intentsPath(repoPath)), 0o755))
+ require.NoError(t, os.WriteFile(intentsPath(repoPath), []byte("\x00\x00\n{\"ref\":\"\",\"original\":\"nope\",\"landed\":\"x\"}\n"), 0o644))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+
+ require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ })
+
+ t.Run("an intent records the head of the log", func(t *testing.T) {
+ repoPath := t.TempDir()
+ seq := appendPlain(t, repoPath, "refs/heads/main", landOID(9))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+
+ intents := intentsOf(t, repoPath)
+ require.Len(t, intents, 1)
+ require.Equal(t, seq, intents[0].Head)
+ })
+}
+
+// TestLandIntentsAreDurable replaces syncFile, so it does not run in parallel.
+func TestLandIntentsAreDurable(t *testing.T) {
+ t.Run("the intents are fsync'd before they return", func(t *testing.T) {
+ repoPath := t.TempDir()
+ var synced []string
+ replaceSyncFile(t, func(f *os.File) error {
+ synced = append(synced, filepath.Base(f.Name()))
+ return f.Sync()
+ })
+
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ require.Contains(t, synced, intentsFileName)
+ })
+
+ t.Run("a failed fsync fails the intent", func(t *testing.T) {
+ repoPath := t.TempDir()
+ replaceSyncFile(t, func(f *os.File) error {
+ if filepath.Base(f.Name()) == intentsFileName {
+ return fmt.Errorf("injected")
+ }
+ return f.Sync()
+ })
+
+ require.ErrorContains(t, RecordLandIntents(repoPath, "refs/heads/main", []Land{{Original: landOID(1), Landed: landOID(2)}}), "injected")
+ })
+}
+
+func TestLandIntentsCompaction(t *testing.T) {
+ t.Parallel()
+
+ repoPath := t.TempDir()
+
+ // Enough intents to compact the file: half of them confirmed by the changelog, a quarter
+ // superseded by a later land, a quarter never committed and never landed again.
+ const n = 2000
+ lands := make([]Land, n)
+ for i := range lands {
+ lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}
+ }
+ require.NoError(t, RecordLandIntents(repoPath, "refs/heads/main", lands))
+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
+
+ var confirmed []Land
+ for i := 0; i < n/2; i++ {
+ confirmed = append(confirmed, lands[i])
+ }
+ for i := n / 2; i < 3*n/4; i++ {
+ confirmed = append(confirmed, Land{Original: lands[i].Original, Landed: landOID(30000 + i)})
+ }
+ updates := make([]Update, 0, len(confirmed))
+ for i := range confirmed {
+ updates = append(updates, Update{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)})
+ }
+ // One entry per land, each carrying its own land.
+ records := make([]Record, len(confirmed))
+ for i, land := range confirmed {
+ records[i] = Record{Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{land}}, Updates: updates[i : i+1]}
+ }
+ _, err := AppendGroup(repoPath, records)
+ require.NoError(t, err)
+
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+
+ kept := intentsOf(t, repoPath)
+ require.Len(t, kept, n/4, "only the intents that the changelog neither confirms nor supersedes are kept")
+ for _, intent := range kept {
+ require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)
+ }
+
+ // The kept intents, more than the size from which a file is compacted, are not rewritten by
+ // every lookup.
+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
+ compacted, err := os.Stat(intentsPath(repoPath))
+ require.NoError(t, err)
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+ again, err := os.Stat(intentsPath(repoPath))
+ require.NoError(t, err)
+ require.True(t, os.SameFile(compacted, again))
+
+ // Every land is still answered as before the compaction, also by a new process.
+ forgetLandsIndex(t, repoPath)
+ for i, land := range lands {
+ records := lookupAll(t, repoPath, "refs/heads/main", land.Original)
+ require.Len(t, records, 1)
+ switch {
+ case i < n/2:
+ require.Equal(t, land.Landed, records[0].Landed)
+ require.NotZero(t, records[0].Sequence)
+ case i < 3*n/4:
+ require.Equal(t, landOID(30000+i), records[0].Landed)
+ default:
+ require.Equal(t, LandRecord{Landed: land.Landed}, records[0])
+ }
+ }
+}
+
+func TestLandsIndexCompaction(t *testing.T) {
+ t.Parallel()
+
+ // The same commit lands again and again (its branch reset past it each time): every record
+ // but the last is superseded.
+ repoPath := t.TempDir()
+ const n = 8000
+ records := make([]Record, n)
+ for i := range records {
+ records[i] = Record{
+ Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{{Original: landOID(1), Landed: landOID(100 + i)}}},
+ Updates: []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(100 + i)}},
+ }
+ }
+ entries, err := AppendGroup(repoPath, records)
+ require.NoError(t, err)
+ last := entries[n-1][0]
+
+ record, found := lookup(t, repoPath, "refs/heads/main", landOID(1))
+ require.True(t, found)
+ require.Equal(t, LandRecord{Sequence: last.Sequence, Landed: landOID(100 + n - 1)}, record)
+
+ lines := landsIndexLines(t, repoPath)
+ require.Len(t, lines, 2, "the compacted file holds one record and its mark")
+ require.Equal(t, last.Sequence, lines[1].Mark)
+
+ forgetLandsIndex(t, repoPath)
+ record, found = lookup(t, repoPath, "refs/heads/main", landOID(1))
+ require.True(t, found)
+ require.Equal(t, LandRecord{Sequence: last.Sequence, Landed: landOID(100 + n - 1)}, record)
+}
+
+func TestLandsIndexOfAReplacedLogFile(t *testing.T) {
+ t.Parallel()
+
+ // The repository is created again at the same path, and its log ends with the same entry as
+ // the old one: the index in memory, whose mark still matches, must not answer for it.
+ repoPath := t.TempDir()
+ appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ appendPlain(t, repoPath, "refs/heads/main", landOID(9))
+ _, found := lookup(t, repoPath, "refs/heads/main", landOID(1))
+ require.True(t, found)
+
+ entries, err := List(repoPath, 0, 10)
+ require.NoError(t, err)
+ entries[0].Lands = nil
+ var buf bytes.Buffer
+ for _, entry := range entries {
+ line, err := json.Marshal(entry)
+ require.NoError(t, err)
+ buf.Write(line)
+ buf.WriteByte('\n')
+ }
+ replacement := Path(repoPath) + ".new"
+ require.NoError(t, os.WriteFile(replacement, buf.Bytes(), 0o644))
+ require.NoError(t, os.Rename(replacement, Path(repoPath)))
+ forgetLog(t, repoPath)
+
+ _, found = lookup(t, repoPath, "refs/heads/main", landOID(1))
+ require.False(t, found)
+}
+
+func TestLandsIndexesAreEvictedLeastRecentlyUsedFirst(t *testing.T) {
+ // The test fills the shared cache of indexes, so it does not run in parallel.
+ landsIndexesMu.Lock()
+ saved := landsIndexes
+ landsIndexes = map[string]*landsIndex{}
+ landsIndexesMu.Unlock()
+ t.Cleanup(func() {
+ landsIndexesMu.Lock()
+ landsIndexes = saved
+ landsIndexesMu.Unlock()
+ })
+
+ repos := make([]string, maxIdleLandsIndexes+1)
+ for i := range repos {
+ repos[i] = t.TempDir()
+ appendLand(t, repos[i], "refs/heads/main", landOID(1), landOID(2))
+ lookup(t, repos[i], "refs/heads/main", landOID(1))
+ }
+ cached := func(repoPath string) bool {
+ landsIndexesMu.Lock()
+ defer landsIndexesMu.Unlock()
+ _, ok := landsIndexes[landsPath(repoPath)]
+ return ok
+ }
+
+ // The first repository was used least recently when the last one came in.
+ require.False(t, cached(repos[0]))
+ for _, repoPath := range repos[1:] {
+ require.True(t, cached(repoPath))
+ }
+
+ // Using the second repository again keeps it; the third is now the least recently used.
+ lookup(t, repos[1], "refs/heads/main", landOID(1))
+ lookup(t, repos[0], "refs/heads/main", landOID(1))
+ require.True(t, cached(repos[1]))
+ require.False(t, cached(repos[2]))
+}
diff --git a/internal/graft/changelog/lands_test.go b/internal/graft/changelog/lands_test.go
index 89b31c0..96c1403 100644
--- a/internal/graft/changelog/lands_test.go
+++ b/internal/graft/changelog/lands_test.go
@@ -40,13 +40,25 @@ func appendPlain(t *testing.T, repoPath, reference, newOID string) uint64 {
return entries[0].Sequence
}
-// lookup looks up a land and fails the test on an error.
-func lookup(t *testing.T, repoPath, reference, original string) (LandRecord, bool) {
+// lookupAll looks up the lands of original on reference and fails the test on an error.
+func lookupAll(t *testing.T, repoPath, reference, original string) []LandRecord {
t.Helper()
- record, found, err := LookupLand(repoPath, reference, original)
+ records, err := LookupLand(repoPath, reference, original)
require.NoError(t, err)
- return record, found
+ return records
+}
+
+// lookup returns the land of original on reference that the changelog recorded.
+func lookup(t *testing.T, repoPath, reference, original string) (LandRecord, bool) {
+ t.Helper()
+
+ for _, record := range lookupAll(t, repoPath, reference, original) {
+ if record.Sequence != 0 {
+ return record, true
+ }
+ }
+ return LandRecord{}, false
}
// forgetLandsIndex drops the in-memory index of the repository, as a new process starts without it.
@@ -77,14 +89,21 @@ func TestValidateLands(t *testing.T) {
})
t.Run("the largest batch with the longest branch name fits", func(t *testing.T) {
- // Every byte of the name is escaped by JSON into 6 bytes, the worst case.
- reference := "refs/heads/" + strings.Repeat("<", 4096-len("refs/heads/"))
+ // The name travels in base64; bytes that are not valid UTF-8 cost no more than others.
+ reference := "refs/heads/" + strings.Repeat("\xff<", (4096-len("refs/heads/"))/2)
lands := make([]Land, MaxLandsPerUpdate)
for i := range lands {
lands[i] = Land{Original: sha256OID(2 * i), Landed: sha256OID(2*i + 1)}
}
require.NoError(t, ValidateLands(reference, lands))
+ require.NoError(t, CheckLandReference(reference))
+ })
+
+ t.Run("a branch name too long for a full batch is refused when the land arrives", func(t *testing.T) {
+ longest := "refs/heads/" + strings.Repeat("a", (MaxLandsSize-fullBatchLandsSize)/4*3-len("refs/heads/"))
+ require.NoError(t, CheckLandReference(longest))
+ require.ErrorContains(t, CheckLandReference(longest+"aaaa"), "is too long to record the identity of a land on it")
})
t.Run("more lands than a batch has", func(t *testing.T) {
@@ -97,7 +116,7 @@ func TestValidateLands(t *testing.T) {
})
t.Run("identities beyond the size bound", func(t *testing.T) {
- reference := "refs/heads/" + strings.Repeat("<", MaxLandsSize/6)
+ reference := "refs/heads/" + strings.Repeat("<", MaxLandsSize/4*3)
require.ErrorContains(t, ValidateLands(reference, []Land{land}), fmt.Sprintf("at most %d", MaxLandsSize))
})
diff --git a/internal/graft/changelog/log.go b/internal/graft/changelog/log.go
index c101eb8..661cdb2 100644
--- a/internal/graft/changelog/log.go
+++ b/internal/graft/changelog/log.go
@@ -206,6 +206,10 @@ type logState struct {
loaded bool
// lastSeq is the sequence of the last valid entry.
lastSeq uint64
+ // lastReference and lastNewOID are the reference and the new value of the last valid entry,
+ // empty when there is none. With lastSeq they identify the last entry (see LogHead).
+ lastReference string
+ lastNewOID string
// validEnd is the offset just after the last complete, valid entry.
validEnd int64
// fileSize is the size of the file when the cache was last updated.
@@ -452,6 +456,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entries
var (
buf bytes.Buffer
points []indexPoint
+ last Entry
)
seq := s.lastSeq
for i, record := range records {
@@ -491,6 +496,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entries
buf.Write(line)
buf.WriteByte('\n')
recorded = append(recorded, entry)
+ last = entry
}
entries[i] = recorded
}
@@ -520,6 +526,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entries
}
s.lastSeq = seq
+ s.lastReference, s.lastNewOID = last.Reference, last.NewOID
s.validEnd += int64(buf.Len())
s.fileSize = s.validEnd
s.loaded = true
@@ -606,6 +613,7 @@ func (s *logState) load(f *os.File, size int64) error {
func (s *logState) rescan(f *os.File, size int64) error {
s.rescans++
s.lastSeq = 0
+ s.lastReference, s.lastNewOID = "", ""
s.validEnd = 0
// The bytes after the last newline are a torn tail, which is not part of the
@@ -635,6 +643,7 @@ func (s *logState) rescan(f *os.File, size int64) error {
var entry Entry
if err := json.Unmarshal(line, &entry); err == nil && entry.Sequence != 0 {
s.lastSeq = entry.Sequence
+ s.lastReference, s.lastNewOID = entry.Reference, entry.NewOID
s.validEnd = nl + 1
break
}
@@ -760,7 +769,7 @@ var truncateFile = func(f *os.File, size int64) error {
}
// durableView returns the end of the durable entries of the log at path and the
-// sequence of the last of them. f is the log file, opened by the caller. Readers
+// last of them (see logHead). f is the log file, opened by the caller. Readers
// read the log only up to that end.
//
// The cached state only moves past entries once they are durable, so while it
@@ -772,46 +781,46 @@ var truncateFile = func(f *os.File, size int64) error {
// page cache are durable once a reader may see them. So are the directory entries
// of the log, which that writer may have crashed before syncing (see
// syncLogDirs), unless this process synced them already.
-func (s *logState) durableView(repoPath, path string, f *os.File) (int64, uint64, error) {
+func (s *logState) durableView(repoPath, path string, f *os.File) (int64, logHead, error) {
s.mu.Lock()
defer s.mu.Unlock()
info, err := f.Stat()
if err != nil {
- return 0, 0, fmt.Errorf("stat changelog: %w", err)
+ return 0, logHead{}, fmt.Errorf("stat changelog: %w", err)
}
if valid, err := s.cacheDescribes(f, info.Size()); err != nil {
- return 0, 0, err
+ return 0, logHead{}, err
} else if valid && s.dirsSynced {
- return s.validEnd, s.lastSeq, nil
+ return s.validEnd, s.head(), nil
}
unlock, err := lockFile(path + lockSuffix)
if err != nil {
- return 0, 0, err
+ return 0, logHead{}, err
}
defer unlock()
if err := syncFile(f); err != nil {
- return 0, 0, fmt.Errorf("syncing changelog: %w", err)
+ return 0, logHead{}, fmt.Errorf("syncing changelog: %w", err)
}
if !s.dirsSynced {
if err := syncLogDirs(repoPath, path); err != nil {
- return 0, 0, err
+ return 0, logHead{}, err
}
s.dirsSynced = true
}
info, err = f.Stat()
if err != nil {
- return 0, 0, fmt.Errorf("stat changelog: %w", err)
+ return 0, logHead{}, fmt.Errorf("stat changelog: %w", err)
}
if err := s.load(f, info.Size()); err != nil {
s.loaded = false
- return 0, 0, err
+ return 0, logHead{}, err
}
- return s.validEnd, s.lastSeq, nil
+ return s.validEnd, s.head(), nil
}
// syncDir fsyncs the directory dir, so that the entries it holds are durable.
@@ -953,6 +962,37 @@ func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {
// AppendGroup). It uses the cached state of the log while it describes the file,
// so its cost does not depend on the size of the log.
func LastSequence(repoPath string) (uint64, error) {
+ head, err := readLogHead(repoPath)
+ if err != nil {
+ return 0, err
+ }
+
+ return head.sequence, nil
+}
+
+// logHead identifies the last durable entry of a log and the file that holds it.
+type logHead struct {
+ // sequence, reference and newOID are those of the last durable entry; the zero value when the
+ // log has none.
+ sequence uint64
+ reference string
+ newOID string
+ // device and inode identify the log file. A log that was replaced, for example with its
+ // repository, is another file.
+ device uint64
+ inode uint64
+}
+
+// head returns the head of the log as the cache describes it, without the file identity. The
+// caller must hold s.mu.
+func (s *logState) head() logHead {
+ return logHead{sequence: s.lastSeq, reference: s.lastReference, newOID: s.lastNewOID}
+}
+
+// readLogHead returns the head of the log of the repository whose git directory is repoPath: the
+// zero value when the log does not exist. Like LastSequence, it uses the cached state of the log
+// while it describes the file.
+func readLogHead(repoPath string) (logHead, error) {
path := Path(repoPath)
state, release := acquireState(path)
@@ -961,18 +1001,28 @@ func LastSequence(repoPath string) (uint64, error) {
f, err := os.Open(path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
- return 0, nil
+ return logHead{}, nil
}
- return 0, fmt.Errorf("opening changelog: %w", err)
+ return logHead{}, fmt.Errorf("opening changelog: %w", err)
}
defer f.Close()
- _, lastSeq, err := state.durableView(repoPath, path, f)
+ info, err := f.Stat()
if err != nil {
- return 0, err
+ return logHead{}, fmt.Errorf("stat changelog: %w", err)
+ }
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return logHead{}, fmt.Errorf("stat changelog: no file identity in %T", info.Sys())
+ }
+
+ _, head, err := state.durableView(repoPath, path, f)
+ if err != nil {
+ return logHead{}, err
}
+ head.device, head.inode = uint64(stat.Dev), stat.Ino
- return lastSeq, nil
+ return head, nil
}
// List returns up to limit entries with a sequence greater than after, in
diff --git a/sdk/land.go b/sdk/land.go
index a044e2d..d949018 100644
--- a/sdk/land.go
+++ b/sdk/land.go
@@ -83,6 +83,14 @@ type LandOptions struct {
// the update lands the change; with ResolveConflicts it may run the conflict
// resolver again, a model included, for the same conflict, and LandResult
// reports the retries (UnavailableRetries), so that this cost is visible.
+//
+// The record of the first land needs a server released after v0.2.0 with its
+// changeset log on ([graft.changelog] not disabled). Against an older server,
+// or one whose changeset log is disabled, the retry of a lost LandRebase replays
+// the change: every commit of it comes out empty and the branch is left
+// unchanged, without AlreadyLanded, unless a later commit reverted the change in
+// between, which the retry then lands again. Set SkipRebaseLands for such
+// servers.
type UnavailableRetry struct {
// Disabled turns the retries off: Land returns Unavailable at once.
Disabled bool