Git designed for AI agents.
PLAN11 re-review: exact intent confirmation, the empty payload, and fixes
Federico Elia53ea79716cac
diff --git a/internal/git/gitcmd/hooks_payload.go b/internal/git/gitcmd/hooks_payload.goindex e99ccc3..99394d1 100644--- a/internal/git/gitcmd/hooks_payload.go+++ b/internal/git/gitcmd/hooks_payload.go@@ -264,7 +264,13 @@ func NewHooksPayload(id, hasIdentity := identity.FromContext(ctx)members, membersTruncated := capChangelogMembers(changelog.BatchFromContext(ctx))- landReference, lands := changelog.LandsFromContext(ctx)+ // The payload holds no branch for an update that lands nothing: an empty slice would not+ // survive the encoding (omitempty), and the decoded payload would differ.+ var landReference []byte+ reference, lands := changelog.LandsFromContext(ctx)+ if reference != "" {+ landReference = []byte(reference)+ }return HooksPayload{Repo: &gitalypb.Repository{@@ -293,7 +299,7 @@ func NewHooksPayload(Rewritten: changelog.IsRewritten(ctx),Members: members,MembersTruncated: membersTruncated,- LandReference: []byte(landReference),+ LandReference: landReference,Lands: lands,MVCCCommit: mvccCommitHandle(ctx, cfg),},diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.goindex ac91f9e..70d5c8d 100644--- a/internal/gitaly/hook/referencetransaction.go+++ b/internal/gitaly/hook/referencetransaction.go@@ -226,7 +226,22 @@ func (m *GitLabHookManager) recordLandIntents(ctx context.Context, payload gitcmreturn fmt.Errorf("resolving repository path: %w", err)}- return changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Lands)+ compact, err := changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Invocation, meta.Lands)+ if err != nil {+ return err+ }++ // The compaction reads the changelog and rewrites the intents: it runs after the hook, not+ // under the locks of the references, and its failure leaves the intents as they are.+ if compact {+ go func() {+ if err := changelog.CompactLandIntents(repoPath); err != nil {+ m.logger.WithError(err).WithField("repository", repoPath).Error("graft: compacting the land intents failed")+ }+ }()+ }++ return nil}// prepareChangesets gives the prepared transaction its place in the order of the changeset log,diff --git a/internal/gitaly/hook/referencetransaction_intents_test.go b/internal/gitaly/hook/referencetransaction_intents_test.gonew file mode 100644index 0000000..155ebdd--- /dev/null+++ b/internal/gitaly/hook/referencetransaction_intents_test.go@@ -0,0 +1,87 @@+package hook++import (+ "context"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/git/gitcmd"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/graft/changelog"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+)++// TestReferenceTransactionHook_recordsLandIntents checks that the prepared phase of an update that+// lands changes records their intents before git may commit it (doc/graft/PLAN11.md, 4.2), that an+// update of another reference records none, and that a failure to record them aborts the update.+func TestReferenceTransactionHook_recordsLandIntents(t *testing.T) {+ t.Parallel()++ land := changelog.Land{Original: strings.Repeat("a", 40), Landed: strings.Repeat("b", 40)}++ // prepare runs the prepared phase of an update of reference whose payload lands land on main,+ // in a new repository, and returns the repository's path and the error of the hook.+ prepare := func(t *testing.T, cfg config.Cfg, reference string, setup func(repoPath string)) (string, error) {+ ctx := testhelper.Context(t)+ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ repoPath, err := config.NewLocator(cfg).GetRepoPath(ctx, repo)+ require.NoError(t, err)+ if setup != nil {+ setup(repoPath)+ }++ landCtx, err := changelog.WithLands(ctx, "refs/heads/main", []changelog.Land{land})+ require.NoError(t, err)+ env, err := gitcmd.NewHooksPayload(landCtx, cfg, repo, gittest.DefaultObjectHash, nil, nil, gitcmd.ReferenceTransactionHook, nil).Env()+ require.NoError(t, err)++ changes := fmt.Sprintf("%s %s %s\n", gittest.DefaultObjectHash.EmptyTreeOID, land.Landed, reference)+ return repoPath, newMembersTestManager(t, cfg).ReferenceTransactionHook(context.Background(), ReferenceTransactionPrepared, []string{env}, strings.NewReader(changes))+ }++ t.Run("an update of the branch of the lands", func(t *testing.T) {+ t.Parallel()++ repoPath, err := prepare(t, testcfg.Build(t), "refs/heads/main", nil)+ require.NoError(t, err)++ records, err := changelog.LookupLand(repoPath, "refs/heads/main", land.Original)+ require.NoError(t, err)+ require.Equal(t, []changelog.LandRecord{{Landed: land.Landed}}, records)+ })++ t.Run("an update of another reference", func(t *testing.T) {+ t.Parallel()++ repoPath, err := prepare(t, testcfg.Build(t), "refs/heads/other", nil)+ require.NoError(t, err)+ require.NoFileExists(t, filepath.Join(repoPath, "graft", "changelog.intents"))+ })++ t.Run("without a changelog", func(t *testing.T) {+ t.Parallel()++ cfg := testcfg.Build(t)+ cfg.Graft.Changelog.Disabled = true+ repoPath, err := prepare(t, cfg, "refs/heads/main", nil)+ require.NoError(t, err)+ require.NoFileExists(t, filepath.Join(repoPath, "graft", "changelog.intents"))+ })++ t.Run("a failure to record the intents aborts the update", func(t *testing.T) {+ t.Parallel()++ // The intents file cannot be opened for writing: a directory is in its place.+ _, err := prepare(t, testcfg.Build(t), "refs/heads/main", func(repoPath string) {+ require.NoError(t, os.MkdirAll(filepath.Join(repoPath, "graft", "changelog.intents"), 0o755))+ })+ require.ErrorContains(t, err, "recording the land intents")+ })+}diff --git a/internal/gitaly/hook/storage_voter.go b/internal/gitaly/hook/storage_voter.goindex a554395..b07e933 100644--- a/internal/gitaly/hook/storage_voter.go+++ b/internal/gitaly/hook/storage_voter.go@@ -105,6 +105,9 @@ func changelogMetaOf(payload gitcmd.HooksPayload, graftMeta gitcmd.ChangelogMetaLandReference: string(graftMeta.LandReference),Lands: graftMeta.Lands,}+ if len(graftMeta.Lands) > 0 {+ meta.LandInvocation = graftMeta.Invocation+ }if meta.Operation == "" {meta.Operation = changelog.UnknownOperation}diff --git a/internal/gitaly/service/operations/land_identity.go b/internal/gitaly/service/operations/land_identity.goindex 6b31413..79d9316 100644--- a/internal/gitaly/service/operations/land_identity.go+++ b/internal/gitaly/service/operations/land_identity.go@@ -85,14 +85,16 @@ func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.RefereonBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())if err != nil {- // The commit does not exist: an intent of an update that was not committed, whose- // objects were never migrated, or a land whose branch was reset past it and whose- // commit was pruned since. Either way the change is not on the branch through it.+ // The commit does not exist: it was pruned since, after the branch was reset past it,+ // or its update was aborted and its objects pruned. The change is not on the branch+ // through it.if errors.As(err, new(localrepo.InvalidCommitError)) {continue}return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)}+ // An update migrates its objects before it is prepared, so the commit of an intent whose+ // update was aborted exists, off the branch, and does not answer.if !onBranch {continue}diff --git a/internal/gitaly/service/operations/land_identity_test.go b/internal/gitaly/service/operations/land_identity_test.goindex 68039a5..02951f8 100644--- a/internal/gitaly/service/operations/land_identity_test.go+++ b/internal/gitaly/service/operations/land_identity_test.go@@ -7,6 +7,7 @@ import ("os""strconv""strings"+ "sync/atomic""testing""github.com/stretchr/testify/require"@@ -15,8 +16,13 @@ import ("github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest""github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/gitaly/transaction""github.com/feder-cr/git-ai/internal/graft/changelog"+ "github.com/feder-cr/git-ai/internal/grpc/metadata""github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testserver"+ "github.com/feder-cr/git-ai/internal/transaction/txinfo"+ "github.com/feder-cr/git-ai/internal/transaction/voting")// TestLandBatch_retryAfterALostResponse lands two changes by rebase in one batch, and then lands@@ -113,9 +119,9 @@ func testUserLandIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.CrequireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, landed)}-// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing records intents whose updates git never-// committed, as an update that was aborted after its prepared phase leaves them: their landed-// commit is not on the branch, so the change lands.+// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing aborts a land after the prepared phase of+// its update, which leaves its intent: the intent's landed commit exists (the objects of an update+// are migrated before it is prepared) but is not on the branch, so the next land lands the change.func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {t.Parallel()@@ -125,19 +131,46 @@ func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {func testUserLandIntentOfAnUncommittedUpdateAnswersNothing(t *testing.T, ctx context.Context) {t.Parallel()- ctx, cfg, client := setupOperationsService(t, ctx)+ txManager := transaction.MockManager{}+ ctx, cfg, client := setupOperationsService(t, ctx, testserver.WithTransactionManager(&txManager))++ ctx, err := txinfo.InjectTransaction(ctx, 1, "node", true)+ require.NoError(t, err)+ ctx = metadata.IncomingToOutgoing(ctx)++ // The repository is set up without the transaction failing.+ txManager.VoteFn = func(context.Context, txinfo.Transaction, voting.Vote, voting.Phase) error { return nil }fx := newRetryFixture(t, ctx, cfg)- // One intent names a commit that does not exist, the other one that is not on the branch.- require.NoError(t, changelog.RecordLandIntents(fx.repoPath, git.DefaultRef.String(), []changelog.Land{- {Original: fx.change.String(), Landed: strings.Repeat("1", len(fx.change))},- {Original: fx.change.String(), Landed: fx.change.String()},- }))+ // The first land fails its vote in the prepared phase of its update, after the hook recorded+ // its intent: git aborts the update. Its objects were migrated before the update, so its+ // landed commit exists, off the branch.+ var failed atomic.Bool+ txManager.VoteFn = func(_ context.Context, _ txinfo.Transaction, _ voting.Vote, phase voting.Phase) error {+ if phase == voting.Prepared && failed.CompareAndSwap(false, true) {+ return errors.New("the transaction lost its quorum")+ }+ return nil+ }++ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)+ _, err = client.UserLand(ctx, request)+ require.Error(t, err)+ require.True(t, failed.Load())+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, fx.tip)- resp, err := client.UserLand(ctx, landRequest(fx.repoProto, git.DefaultBranch, fx.change))+ records, err := changelog.LookupLand(fx.repoPath, git.DefaultRef.String(), fx.change.String())require.NoError(t, err)- require.False(t, resp.GetAlreadyLanded())- require.True(t, resp.GetRewritten())+ require.Len(t, records, 1, "the aborted update left its intent")+ aborted := records[0].Landed+ require.Equal(t, "commit\n", string(gittest.Exec(t, cfg, "-C", fx.repoPath, "cat-file", "-t", aborted)), "the aborted commit exists")++ // The intent's commit is not on the branch, so the land lands the change.+ second, err := client.UserLand(ctx, request)+ require.NoError(t, err)+ require.False(t, second.GetAlreadyLanded())+ require.True(t, second.GetRewritten())+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, git.ObjectID(second.GetLandedCommitId()))}// TestUserLand_identityOfABranchWhoseNameIsNotUTF8 lands a change by rebase on a branch whose name@@ -247,6 +280,58 @@ func TestUserLand_branchNameTooLongForTheIdentity(t *testing.T) {_, err := client.UserLand(ctx, landRequest(fx.repoProto, branch, fx.change))testhelper.RequireGrpcCode(t, err, codes.InvalidArgument)require.ErrorContains(t, err, "is too long to record the identity of a land on it")+ require.Less(t, len(err.Error()), 512, "the error quotes the beginning of the name, not all of it")+}++// TestLandBatch_intentAnswersWhatTheChangelogMisses lands two changes in one batch, loses the+// changelog entry of their update, and lands them again: the intents the hook recorded for the+// batch's update answer.+func TestLandBatch_intentAnswersWhatTheChangelogMisses(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testLandBatchIntentAnswersWhatTheChangelogMisses)+}++func testLandBatchIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.Context) {+ t.Parallel()++ ctx, cfg, client := setupAgentLandService(t, ctx, agentBatchingGraft(nil))+ repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg)++ baseEntry := gittest.TreeEntry{Mode: "100644", Path: "base", Content: "base"}+ base := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch), gittest.WithTreeEntries(baseEntry))+ gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch), gittest.WithParents(base),+ gittest.WithTreeEntries(baseEntry, gittest.TreeEntry{Mode: "100644", Path: "moved", Content: "moved"}))+ change := func(name string) git.ObjectID {+ return gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(base),+ gittest.WithTreeEntries(baseEntry, gittest.TreeEntry{Mode: "100644", Path: name, Content: name}))+ }+ a, b := change("a"), change("b")++ first, errs := landInOrder(ctx, client, repoProto,+ agentLanding{agent: "agent-a", commit: a},+ agentLanding{agent: "agent-b", commit: b},+ )+ for i := range errs {+ require.NoError(t, errs[i])+ require.Equal(t, int32(2), first[i].GetBatchSize(), "both changes land in one batch")+ }+ tip := gittest.ResolveRevision(t, cfg, repoPath, git.DefaultRef.String())++ // The changelog lost the entry of the batch's update, its only entry.+ require.NoError(t, os.Remove(changelog.Path(repoPath)))++ second, errs := landInOrder(ctx, client, repoProto,+ agentLanding{agent: "agent-a", commit: a},+ agentLanding{agent: "agent-b", commit: b},+ )+ for i := range errs {+ require.NoError(t, errs[i])+ require.True(t, second[i].GetAlreadyLanded())+ require.Equal(t, first[i].GetLandedCommitId(), second[i].GetLandedCommitId())+ require.Zero(t, second[i].GetLandedSequence())+ }+ requireBranchAt(t, cfg, repoPath, git.DefaultBranch, tip)}// TestLandBatchExecutorRetryInTheBatchOfItsFirstLand puts a request and its retry in one batch:diff --git a/internal/graft/changelog/lands.go b/internal/graft/changelog/lands.goindex c677397..1f54f7e 100644--- a/internal/graft/changelog/lands.go+++ b/internal/graft/changelog/lands.go@@ -70,7 +70,8 @@ var fullBatchLandsSize = func() int {// too long for them cannot be landed on, since a land would not record its identity.func CheckLandReference(reference string) error {if size := fullBatchLandsSize + base64.StdEncoding.EncodedLen(len(reference)); size > MaxLandsSize {- return fmt.Errorf("the name of branch %q is too long to record the identity of a land on it: %d bytes, at most %d",+ // The name is too long to be quoted whole: its first bytes say which branch it is.+ return fmt.Errorf("the name of branch %.64q... is too long to record the identity of a land on it: %d bytes, at most %d",reference, len(reference), (MaxLandsSize-fullBatchLandsSize)/4*3)}return nil@@ -163,3 +164,13 @@ func LandsOf(meta Meta, reference string) []Land {return append([]Land(nil), meta.Lands...)}++// LandInvocationOf returns the invocation that the changelog entry of an update of reference+// carries under meta with its lands, see Entry.LandInvocation: empty when it carries no lands.+func LandInvocationOf(meta Meta, reference string) string {+ if len(LandsOf(meta, reference)) == 0 {+ return ""+ }++ return meta.LandInvocation+}diff --git a/internal/graft/changelog/lands_index.go b/internal/graft/changelog/lands_index.goindex 222e12a..45bcd4b 100644--- a/internal/graft/changelog/lands_index.go+++ b/internal/graft/changelog/lands_index.go@@ -101,7 +101,10 @@ type landsIndex struct {// logDevice and logInode identify the log file that records was read from.logDevice, logInode uint64// records maps the lands of the log entries up to mark to their last record.- records map[landsIndexKey]LandRecord+ records map[landsIndexKey]indexRecord+ // confirmed holds the invocations of the land updates whose changelog entries the index read+ // (Entry.LandInvocation): their intents may go. An invocation leaves it with its intents.+ confirmed map[string]struct{}// recordLines is the number of record lines of the index file, superseded ones included.recordLines int// fileSize is the size of the index file when this process last read or wrote it. A file of@@ -112,10 +115,21 @@ type landsIndex struct {// known of that file.intents map[landsIndexKey][]landIntentintentsFile intentsFile- // intentsKept is the size of the intents file after its last compaction by this process. The- // file is compacted again once it doubled, so that intents that cannot go yet are not- // rewritten by every lookup.- intentsKept int64+}++// indexRecord is the last record of a land in an index.+type indexRecord struct {+ LandRecord+ // invocation identifies the update of the record, see Entry.LandInvocation.+ invocation string+}++// forget drops what the index knows, after a catch-up that failed: the next lookup reads the+// index and the intents again. The caller must hold i.mu.+func (i *landsIndex) forget() {+ i.loaded = false+ i.intents = nil+ i.intentsFile = intentsFile{}}var (@@ -194,10 +208,12 @@ func landsPath(repoPath string) string {// LookupLand returns what the repository whose git directory is repoPath knows of the lands of// original on reference (doc/graft/PLAN11.md): first the last land that the changelog recorded,// with the sequence of its entry and the commit it landed as, then the land intents (see-// landIntent) of lands that the changelog does not record, with a zero sequence. A land answers-// only when its landed commit is on the branch, which the caller checks. The result is empty when-// nothing is known. The lookup reads the durable entries of the log (see AppendGroup) and the-// intents that the index does not cover yet, and adds them to the index first.+// landIntent) of other landed commits, with a zero sequence. A land answers only when its landed+// commit is on the branch, which the caller checks: an intent whose commit is on the branch proves+// the land by itself, whatever the changelog recorded since, so no intent is left out but those of+// the recorded commit. The result is empty when nothing is known. The lookup reads the durable+// entries of the log (see AppendGroup) and the intents that the index does not cover yet, and adds+// them to the index first.func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {index, release := acquireLandsIndex(landsPath(repoPath))defer release()@@ -207,21 +223,22 @@ func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {if err := index.catchUp(repoPath); err != nil {// What the failed catch-up did to the index is not known, so the next lookup reads it again.- index.loaded = false- index.intentsFile = intentsFile{}+ index.forget()return nil, err}key := landsIndexKey{reference: reference, original: original}var lands []LandRecord- confirmed, found := index.records[key]+ recorded, found := index.records[key]if found {- lands = append(lands, confirmed)+ lands = append(lands, recorded.LandRecord)}+ seen := map[string]bool{recorded.Landed: found}for _, intent := range index.intents[key] {- if found && (intent.Landed == confirmed.Landed || confirmed.Sequence > intent.Head) {+ if seen[intent.Landed] {continue}+ seen[intent.Landed] = truelands = append(lands, LandRecord{Landed: intent.Landed})}@@ -248,7 +265,10 @@ func (i *landsIndex) estimate() int64 {const overhead = 96var bytes int64for key, record := range i.records {- bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)) + overhead+ bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)+len(record.invocation)) + overhead+ }+ for invocation := range i.confirmed {+ bytes += int64(len(invocation)) + overhead}for key, intents := range i.intents {for _, intent := range intents {@@ -333,7 +353,7 @@ func (i *landsIndex) catchUpLog(repoPath string) error {}for _, record := range block.records {- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}+ i.add(record)}i.recordLines += len(block.records)i.mark = block.mark@@ -356,7 +376,7 @@ func (i *landsIndex) catchUpLog(repoPath string) error {func (i *landsIndex) compact(path string) error {block := landsBlock{mark: i.mark}for key, record := range i.records {- block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed})+ block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed, invocation: record.invocation})}sort.Slice(block.records, func(a, b int) bool { return block.records[a].sequence < block.records[b].sequence })@@ -398,7 +418,7 @@ func (i *landsIndex) catchUpIntents(repoPath string) error {}if reset {// The file is not the one that was read before, and the intents read before are not- // its: they are dropped, and the file is read again from its start.+ // its: they are dropped, and the file is read from its start.i.intents = map[landsIndexKey][]landIntent{}if known, _, err = readIntents(path, intentsFile{}, add); err != nil {return err@@ -406,40 +426,72 @@ func (i *landsIndex) catchUpIntents(repoPath string) error {}i.intentsFile = known- if known.offset < compactIntentsMinSize || known.offset < 2*i.intentsKept {+ // An intent may go once the changelog entry of its own update is durable. The file is+ // compacted from compactIntentsMinSize on, when at least half of its intents may go, so that+ // intents that must stay are not rewritten again and again.+ confirmed := func(intent landIntent) bool {+ _, ok := i.confirmed[intent.Invocation]+ return intent.Invocation != "" && ok+ }+ total, droppable := 0, 0+ for _, intents := range i.intents {+ for _, intent := range intents {+ total+++ if confirmed(intent) {+ droppable+++ }+ }+ }+ if known.offset < compactIntentsMinSize || 2*droppable < total {return nil}- // An intent may go once the changelog confirms it (a record of its land with its landed- // commit) or supersedes it (a later record of the same original commit on the branch).- if err := compactIntents(repoPath, path, func(intent landIntent) bool {- record, found := i.records[landsIndexKey{reference: string(intent.Reference), original: intent.Original}]- return found && (record.Landed == intent.Landed || record.Sequence > intent.Head)- }); err != nil {+ if err := compactIntents(repoPath, path, confirmed); err != nil {return err}+ // The invocations of the dropped intents are not needed any more.+ for _, intents := range i.intents {+ for _, intent := range intents {+ if confirmed(intent) {+ delete(i.confirmed, intent.Invocation)+ }+ }+ }+i.intents = map[landsIndexKey][]landIntent{}known, _, err = readIntents(path, intentsFile{}, add)if err != nil {return err}i.intentsFile = known- i.intentsKept = known.offsetreturn nil}+// add records a land in the index: as the last record of its original commit on its branch, and+// its invocation as confirmed. The caller must hold i.mu.+func (i *landsIndex) add(record landsBlockRecord) {+ i.records[landsIndexKey{reference: record.reference, original: record.original}] = indexRecord{+ LandRecord: LandRecord{Sequence: record.sequence, Landed: record.landed},+ invocation: record.invocation,+ }+ if record.invocation != "" {+ i.confirmed[record.invocation] = struct{}{}+ }+}+// errLandsIndexStale is returned by scanLog when the log entry at the mark of the index is not the// one the mark names.var errLandsIndexStale = errors.New("the land index does not describe the log")// landsBlockRecord is a record of a block, see landsBlock.type landsBlockRecord struct {- sequence uint64- reference string- original string- landed string+ sequence uint64+ reference string+ original string+ landed string+ invocation string}// landsBlock is the records of the lands of the log entries after the mark of an index, and the@@ -472,10 +524,11 @@ func (i *landsIndex) scanLog(repoPath string) (landsBlock, error) {for _, land := range entry.Lands {block.records = append(block.records, landsBlockRecord{- sequence: entry.Sequence,- reference: entry.Reference,- original: land.Original,- landed: land.Landed,+ sequence: entry.Sequence,+ reference: entry.Reference,+ original: land.Original,+ landed: land.Landed,+ invocation: entry.LandInvocation,})}block.mark = landsMark{sequence: entry.Sequence, reference: entry.Reference, newOID: entry.NewOID}@@ -500,6 +553,8 @@ type landsLine struct {Reference []byte `json:"ref"`Original string `json:"original,omitempty"`Landed string `json:"landed,omitempty"`+ // Invocation identifies the update of the record, see Entry.LandInvocation.+ Invocation string `json:"inv,omitempty"`// Mark is the sequence of the log entry that the mark names; Reference and NewOID are its// reference and new value, CRC the CRC-32 (IEEE) of the record lines of the block.Mark uint64 `json:"mark,omitempty"`@@ -512,10 +567,11 @@ func (b landsBlock) encode() ([]byte, error) {var buf bytes.Bufferfor _, record := range b.records {line, err := json.Marshal(landsLine{- Sequence: record.sequence,- Reference: []byte(record.reference),- Original: record.original,- Landed: record.landed,+ Sequence: record.sequence,+ Reference: []byte(record.reference),+ Original: record.original,+ Landed: record.landed,+ Invocation: record.invocation,})if err != nil {return nil, fmt.Errorf("encoding land index record: %w", err)@@ -542,7 +598,8 @@ func (b landsBlock) encode() ([]byte, error) {// reset empties the in-memory index of a file of the given size.func (i *landsIndex) reset(fileSize int64) {i.mark = landsMark{}- i.records = map[landsIndexKey]LandRecord{}+ i.records = map[landsIndexKey]indexRecord{}+ i.confirmed = map[string]struct{}{}i.recordLines = 0i.fileSize = fileSizei.loaded = false@@ -585,10 +642,11 @@ func (i *landsIndex) load(f *os.File) error {break}pending = append(pending, landsBlockRecord{- sequence: decoded.Sequence,- reference: string(decoded.Reference),- original: decoded.Original,- landed: decoded.Landed,+ sequence: decoded.Sequence,+ reference: string(decoded.Reference),+ original: decoded.Original,+ landed: decoded.Landed,+ invocation: decoded.Invocation,})crc.Write(line)continue@@ -600,7 +658,7 @@ func (i *landsIndex) load(f *os.File) error {}for _, record := range pending {- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}+ i.add(record)}i.recordLines += len(pending)i.mark = landsMark{sequence: decoded.Mark, reference: string(decoded.Reference), newOID: decoded.NewOID}diff --git a/internal/graft/changelog/lands_intents.go b/internal/graft/changelog/lands_intents.goindex 772b8f8..e623ab8 100644--- a/internal/graft/changelog/lands_intents.go+++ b/internal/graft/changelog/lands_intents.go@@ -28,10 +28,10 @@ const intentsFileName = "changelog.intents"// committed answers nothing.//// The file is JSON lines, appended under the lock file of the intents and fsync'd before the-// prepared phase ends. Unlike the index it is not derived data. An intent is dropped by a-// compaction once the changelog confirms it (a record of its land with its landed commit) or-// supersedes it (a later record of the same original commit on the branch); the intent of an-// update that was never committed and never landed again stays.+// prepared phase ends. Unlike the index it is not derived data. An intent carries the invocation+// of its update, and so does the changelog entry that the update writes once it is committed+// (Entry.LandInvocation): a compaction drops an intent once the entry of its own update is+// durable, and only then. The intent of an update that was never committed stays.// landIntent is one line of the intents file.type landIntent struct {@@ -40,10 +40,8 @@ type landIntent struct {Reference []byte `json:"ref"`Original string `json:"original"`Landed string `json:"landed"`- // Head is the sequence of the last durable changelog entry when the intent was recorded. The- // entry of the land, if any, comes after it, so a record of the same commit at a later- // sequence confirms or supersedes the intent.- Head uint64 `json:"head"`+ // Invocation identifies the update that recorded the intent, see Entry.LandInvocation.+ Invocation string `json:"invocation"`}// intentsPath returns the path of the intents file of the repository whose git directory is@@ -52,75 +50,145 @@ func intentsPath(repoPath string) string {return filepath.Join(repoPath, dirName, intentsFileName)}-// intentDirsSynced holds the intents files of this process whose directory entries are durable,-// see RecordLandIntents.-var intentDirsSynced sync.Map+// intentsFileKey identifies an intents file: its path, and the file at that path.+type intentsFileKey struct {+ path string+ device, inode uint64+}-// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the-// update of reference about to be committed lands lands (see the comment above landIntent). It-// returns once the intents are fsync'd; an error must abort the update, which would otherwise be-// committed without its identity.-func RecordLandIntents(repoPath, reference string, lands []Land) error {- head, err := readLogHead(repoPath)- if err != nil {- return err- }+var (+ // intentDirsSynced holds the intents files of this process whose directory entries are+ // durable, see RecordLandIntents. A file created again at the same path, as with a+ // repository created again, is another file.+ intentDirsSynced sync.Map+ // intentsCompactionSize holds, by path, the size of the intents file when RecordLandIntents+ // last asked for a compaction, see compactionDue.+ intentsCompactionSize sync.Map+)+// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the+// update of reference about to be committed lands lands, as the update identified by invocation+// (see the comment above landIntent). It returns once the intents are fsync'd; an error must abort+// the update, which would otherwise be committed without its identity. compact reports that the+// file grew so that its compaction is due: the caller runs CompactLandIntents, outside the locks+// of the update.+func RecordLandIntents(repoPath, reference, invocation string, lands []Land) (compact bool, err error) {path := intentsPath(repoPath)if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {- return fmt.Errorf("creating changelog directory: %w", err)+ return false, fmt.Errorf("creating changelog directory: %w", err)}unlock, err := lockFile(path + lockSuffix)if err != nil {- return err+ return false, err}defer unlock()+ _, statErr := os.Stat(path)+ created := errors.Is(statErr, fs.ErrNotExist)+f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)if err != nil {- return fmt.Errorf("opening land intents: %w", err)+ return false, fmt.Errorf("opening land intents: %w", err)}defer f.Close()+ info, err := f.Stat()+ if err != nil {+ return false, fmt.Errorf("stat land intents: %w", err)+ }+ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return false, fmt.Errorf("stat land intents: no file identity in %T", info.Sys())+ }+// The bytes after the last newline are a torn line of a writer that crashed before its fsync,// whose update was therefore never committed. They are cut off, or the next line would be// joined to them.end, err := endOfLastLine(f)if err != nil {- return err+ return false, err}var buf bytes.Bufferfor _, land := range lands {- line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Head: head.sequence})+ line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Invocation: invocation})if err != nil {- return fmt.Errorf("encoding land intent: %w", err)+ return false, fmt.Errorf("encoding land intent: %w", err)}buf.Write(line)buf.WriteByte('\n')}if err := truncateFile(f, end); err != nil {- return fmt.Errorf("truncating land intents: %w", err)+ return false, fmt.Errorf("truncating land intents: %w", err)}if _, err := f.WriteAt(buf.Bytes(), end); err != nil {- return fmt.Errorf("writing land intents: %w", err)+ return false, fmt.Errorf("writing land intents: %w", err)}if err := syncFile(f); err != nil {- return fmt.Errorf("syncing land intents: %w", err)+ return false, fmt.Errorf("syncing land intents: %w", err)}- // The intents are durable only once the directory entries of their file are, which a writer- // that created the file may have crashed before syncing: the first write of this process- // syncs them.- if _, synced := intentDirsSynced.Load(path); !synced {+ // The intents are durable only once the directory entries of their file are: when this write+ // created the file, and on the first write of this process to the file, which a writer that+ // created it may have crashed before syncing.+ key := intentsFileKey{path: path, device: uint64(stat.Dev), inode: stat.Ino}+ if _, synced := intentDirsSynced.Load(key); created || !synced {if err := syncLogDirs(repoPath, path); err != nil {- return err+ return false, err}- intentDirsSynced.Store(path, struct{}{})+ intentDirsSynced.Store(key, struct{}{})}+ return compactionDue(path, end+int64(buf.Len())), nil+}++// compactionDue reports whether an intents file of the given size is due for a compaction: from+// compactIntentsMinSize on, each time it doubled since the last time it was due, or since the last+// CompactLandIntents left it, in this process.+func compactionDue(path string, size int64) bool {+ if size < compactIntentsMinSize {+ return false+ }++ last, _ := intentsCompactionSize.Load(path)+ if lastSize, ok := last.(int64); ok && size < 2*lastSize {+ return false+ }++ intentsCompactionSize.Store(path, size)+ return true+}++// compactionsRunning holds the repositories whose intents CompactLandIntents compacts right now.+var compactionsRunning sync.Map++// CompactLandIntents brings the land index of the repository whose git directory is repoPath up to+// its changelog and intents, which compacts the intents file when enough of its intents are+// confirmed (see landsIndex.catchUpIntents). RecordLandIntents asks for it, so that the intents of+// a repository whose lands never look up an earlier land (fast-forwards only) are compacted too. A+// call while another one runs for the repository returns at once.+func CompactLandIntents(repoPath string) error {+ if _, running := compactionsRunning.LoadOrStore(repoPath, struct{}{}); running {+ return nil+ }+ defer compactionsRunning.Delete(repoPath)++ index, release := acquireLandsIndex(landsPath(repoPath))+ defer release()++ index.mu.Lock()+ defer index.mu.Unlock()++ if err := index.catchUp(repoPath); err != nil {+ index.forget()+ return err+ }++ // The next compaction is due once the file doubled from what this one left.+ intentsCompactionSize.Store(intentsPath(repoPath), index.intentsFile.offset)+return nil}@@ -148,10 +216,11 @@ type intentsFile struct {}// readIntents reads the complete lines of the intents file at path after known, calling add for-// each intent. It returns what the reader knows of the file after the read. When the file is not-// the one known (it was replaced) or is shorter (it was replaced or cut), it is read from the-// start and reset is true: the intents read before do not describe it any more. A missing file has-// no intents.+// each intent, and returns what the reader knows of the file after the read. A reader that knows+// nothing of the file (the zero known) reads it from its start. When the file is not the one known+// (it was replaced) or is shorter (it was replaced or cut), nothing is read and reset is true: the+// intents read before do not describe it, and the caller drops them and reads it again knowing+// nothing. A missing file has no intents.func readIntents(path string, known intentsFile, add func(landIntent)) (intentsFile, bool, error) {f, err := os.Open(path)if err != nil {@@ -172,9 +241,8 @@ func readIntents(path string, known intentsFile, add func(landIntent)) (intentsF}current := intentsFile{device: uint64(stat.Dev), inode: stat.Ino, offset: known.offset}- reset := current.device != known.device || current.inode != known.inode || info.Size() < known.offset- if reset {- current.offset = 0+ if known != (intentsFile{}) && (current.device != known.device || current.inode != known.inode || info.Size() < known.offset) {+ return intentsFile{}, true, nil}r := bufio.NewReader(io.NewSectionReader(f, current.offset, info.Size()-current.offset))@@ -199,7 +267,7 @@ func readIntents(path string, known intentsFile, add func(landIntent)) (intentsFadd(intent)}- return current, reset, nil+ return current, false, nil}// compactIntents rewrites the intents file at path without the intents that drop says may go,diff --git a/internal/graft/changelog/lands_intents_test.go b/internal/graft/changelog/lands_intents_test.goindex f42268a..8cdbfa3 100644--- a/internal/graft/changelog/lands_intents_test.go+++ b/internal/graft/changelog/lands_intents_test.go@@ -12,10 +12,26 @@ import ("github.com/stretchr/testify/require")-// recordIntent records the intent of a land of original on reference as landed.+// invocationOf returns the invocation that the tests give the update of the land of original.+func invocationOf(original string) string { return "inv-" + original }++// recordIntent records the intent of a land of original on reference as landed, by the update+// invocationOf(original).func recordIntent(t *testing.T, repoPath, reference, original, landed string) {t.Helper()- require.NoError(t, RecordLandIntents(repoPath, reference, []Land{{Original: original, Landed: landed}}))+ _, err := RecordLandIntents(repoPath, reference, invocationOf(original), []Land{{Original: original, Landed: landed}})+ require.NoError(t, err)+}++// confirmLand records the changelog entry of the update invocation, which landed original as+// landed on reference.+func confirmLand(t *testing.T, repoPath, reference, original, landed, invocation string) uint64 {+ t.Helper()++ entries, err := Append(repoPath, Meta{Operation: "land", LandReference: reference, LandInvocation: invocation, Lands: []Land{{Original: original, Landed: landed}}},+ []Update{{Reference: reference, OldOID: landOID(0), NewOID: landed}})+ require.NoError(t, err)+ return entries[0].Sequence}// intentsOf returns the intents in the intents file of the repository.@@ -43,26 +59,28 @@ func TestLookupLandAnswersFromIntents(t *testing.T) {require.Empty(t, lookupAll(t, repoPath, "refs/heads/other", landOID(1)))})- t.Run("an intent that the changelog confirms", func(t *testing.T) {+ t.Run("an intent of the recorded commit", func(t *testing.T) {repoPath := t.TempDir()recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))- seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ seq := confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), invocationOf(landOID(1)))require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))})- t.Run("an intent that a later land supersedes", func(t *testing.T) {- // The update of the intent was not committed, and the change was landed again.+ t.Run("an intent of another commit than the recorded one", func(t *testing.T) {+ // Review m1: land A of X is committed and its entry is pending; the branch is reset; land+ // B of X is committed with its intent; A's entry is written, and B's is lost. The intent+ // of B is not left out because a record came after it: it proves B by itself, when its+ // commit is on the branch.repoPath := t.TempDir()- recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))- seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(3))+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))+ seq := confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), "inv-a")- require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(3)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))+ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}, {Landed: landOID(3)}},+ lookupAll(t, repoPath, "refs/heads/main", landOID(1)))})t.Run("an intent after the recorded land", func(t *testing.T) {- // The change landed, the branch was reset past it, and its new land was committed- // without the changelog recording it: both are known, the record first.repoPath := t.TempDir()seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))@@ -87,6 +105,17 @@ func TestLookupLandAnswersFromIntents(t *testing.T) {recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))})++ t.Run("the entry carries the invocation of its update", func(t *testing.T) {+ repoPath := t.TempDir()+ confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), "inv-x")+ appendPlain(t, repoPath, "refs/heads/main", landOID(9))++ entries, err := List(repoPath, 0, 10)+ require.NoError(t, err)+ require.Equal(t, "inv-x", entries[0].LandInvocation)+ require.Empty(t, entries[1].LandInvocation, "an entry without lands has no invocation")+ })}func TestLandIntentsFile(t *testing.T) {@@ -117,18 +146,30 @@ func TestLandIntentsFile(t *testing.T) {require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))})- t.Run("an intent records the head of the log", func(t *testing.T) {+ t.Run("an intent carries the invocation of its update", func(t *testing.T) {repoPath := t.TempDir()- seq := appendPlain(t, repoPath, "refs/heads/main", landOID(9))recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))intents := intentsOf(t, repoPath)require.Len(t, intents, 1)- require.Equal(t, seq, intents[0].Head)+ require.Equal(t, invocationOf(landOID(1)), intents[0].Invocation)+ })++ t.Run("a reader that knows nothing reads the file once", func(t *testing.T) {+ repoPath := t.TempDir()+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))++ read := 0+ known, reset, err := readIntents(intentsPath(repoPath), intentsFile{}, func(landIntent) { read++ })+ require.NoError(t, err)+ require.False(t, reset)+ require.Equal(t, 2, read)+ require.Equal(t, fileSize(t, intentsPath(repoPath)), known.offset)})}-// TestLandIntentsAreDurable replaces syncFile, so it does not run in parallel.+// TestLandIntentsAreDurable replaces syncFile and syncDir, so it does not run in parallel.func TestLandIntentsAreDurable(t *testing.T) {t.Run("the intents are fsync'd before they return", func(t *testing.T) {repoPath := t.TempDir()@@ -151,77 +192,137 @@ func TestLandIntentsAreDurable(t *testing.T) {return f.Sync()})- require.ErrorContains(t, RecordLandIntents(repoPath, "refs/heads/main", []Land{{Original: landOID(1), Landed: landOID(2)}}), "injected")+ _, err := RecordLandIntents(repoPath, "refs/heads/main", "inv", []Land{{Original: landOID(1), Landed: landOID(2)}})+ require.ErrorContains(t, err, "injected")+ })++ t.Run("the directories are synced for a file created again", func(t *testing.T) {+ // Review m3: the repository is removed and created again at the same path. The intents+ // file of the new repository is another file, whose directory entries are synced.+ repoPath := t.TempDir()+ dirSyncs := 0+ original := syncDir+ syncDir = func(dir string) error {+ dirSyncs+++ return original(dir)+ }+ t.Cleanup(func() { syncDir = original })++ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ require.Positive(t, dirSyncs, "the write that created the file syncs the directories")++ dirSyncs = 0+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))+ require.Zero(t, dirSyncs, "a later write to the same file does not")++ require.NoError(t, os.RemoveAll(filepath.Join(repoPath, dirName)))+ recordIntent(t, repoPath, "refs/heads/main", landOID(5), landOID(6))+ require.Positive(t, dirSyncs, "the write that created the file again syncs the directories")})}func TestLandIntentsCompaction(t *testing.T) {t.Parallel()- repoPath := t.TempDir()+ // recordAndConfirm records n intents in one update each, and the changelog entries of the+ // updates of the first confirmed of them. The entries of the next superseded intents' commits+ // are written by other updates, with other landed commits: they do not confirm the intents.+ recordAndConfirm := func(t *testing.T, repoPath string, n, confirmed, superseded int) []Land {+ lands := make([]Land, n)+ for i := range lands {+ lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}+ _, err := RecordLandIntents(repoPath, "refs/heads/main", invocationOf(lands[i].Original), lands[i:i+1])+ require.NoError(t, err)+ }- // Enough intents to compact the file: half of them confirmed by the changelog, a quarter- // superseded by a later land, a quarter never committed and never landed again.- const n = 2000- lands := make([]Land, n)- for i := range lands {- lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}- }- require.NoError(t, RecordLandIntents(repoPath, "refs/heads/main", lands))- require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))+ var records []Record+ for i := 0; i < confirmed+superseded; i++ {+ land, invocation := lands[i], invocationOf(lands[i].Original)+ if i >= confirmed {+ land, invocation = Land{Original: lands[i].Original, Landed: landOID(30000 + i)}, "inv-other"+ }+ records = append(records, Record{+ Meta: Meta{Operation: "land", LandReference: "refs/heads/main", LandInvocation: invocation, Lands: []Land{land}},+ Updates: []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)}},+ })+ }+ _, err := AppendGroup(repoPath, records)+ require.NoError(t, err)- var confirmed []Land- for i := 0; i < n/2; i++ {- confirmed = append(confirmed, lands[i])- }- for i := n / 2; i < 3*n/4; i++ {- confirmed = append(confirmed, Land{Original: lands[i].Original, Landed: landOID(30000 + i)})+ return lands}- updates := make([]Update, 0, len(confirmed))- for i := range confirmed {- updates = append(updates, Update{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)})- }- // One entry per land, each carrying its own land.- records := make([]Record, len(confirmed))- for i, land := range confirmed {- records[i] = Record{Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{land}}, Updates: updates[i : i+1]}- }- _, err := AppendGroup(repoPath, records)- require.NoError(t, err)- lookupAll(t, repoPath, "refs/heads/main", landOID(10000))+ t.Run("an intent goes once the entry of its own update is durable", func(t *testing.T) {+ repoPath := t.TempDir()+ const n = 2000+ lands := recordAndConfirm(t, repoPath, n, 3*n/4, n/8)+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))- kept := intentsOf(t, repoPath)- require.Len(t, kept, n/4, "only the intents that the changelog neither confirms nor supersedes are kept")- for _, intent := range kept {- require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)- }+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))- // The kept intents, more than the size from which a file is compacted, are not rewritten by- // every lookup.- require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))- compacted, err := os.Stat(intentsPath(repoPath))- require.NoError(t, err)- lookupAll(t, repoPath, "refs/heads/main", landOID(10000))- again, err := os.Stat(intentsPath(repoPath))- require.NoError(t, err)- require.True(t, os.SameFile(compacted, again))+ kept := intentsOf(t, repoPath)+ require.Len(t, kept, n/4, "the intents whose own update has no entry stay, also those of a recorded commit")+ for _, intent := range kept {+ require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)+ }- // Every land is still answered as before the compaction, also by a new process.- forgetLandsIndex(t, repoPath)- for i, land := range lands {- records := lookupAll(t, repoPath, "refs/heads/main", land.Original)- require.Len(t, records, 1)- switch {- case i < n/2:- require.Equal(t, land.Landed, records[0].Landed)- require.NotZero(t, records[0].Sequence)- case i < 3*n/4:- require.Equal(t, landOID(30000+i), records[0].Landed)- default:- require.Equal(t, LandRecord{Landed: land.Landed}, records[0])+ // Every land is still answered as before the compaction, also by a new process.+ forgetLandsIndex(t, repoPath)+ for i, land := range lands {+ records := lookupAll(t, repoPath, "refs/heads/main", land.Original)+ switch {+ case i < 3*n/4:+ require.Len(t, records, 1)+ require.Equal(t, land.Landed, records[0].Landed)+ require.NotZero(t, records[0].Sequence)+ case i < 3*n/4+n/8:+ require.Equal(t, []LandRecord{{Sequence: records[0].Sequence, Landed: landOID(30000 + i)}, {Landed: land.Landed}}, records)+ default:+ require.Equal(t, []LandRecord{{Landed: land.Landed}}, records)+ }}- }+ })++ t.Run("intents that must stay are not rewritten again", func(t *testing.T) {+ // Review m4: after a compaction, also in a new process, the intents that stay are not+ // rewritten while too few of them may go.+ repoPath := t.TempDir()+ const n = 2000+ recordAndConfirm(t, repoPath, n, n/2, 0)+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))+ require.Len(t, intentsOf(t, repoPath), n/2)+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))++ compacted, err := os.Stat(intentsPath(repoPath))+ require.NoError(t, err)+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))+ forgetLandsIndex(t, repoPath)+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))+ again, err := os.Stat(intentsPath(repoPath))+ require.NoError(t, err)+ require.True(t, os.SameFile(compacted, again))+ })++ t.Run("the intents of lands that never look up are compacted too", func(t *testing.T) {+ // Review m4: fast-forward lands never look up an earlier land. The append asks for the+ // compaction once the file grew, and CompactLandIntents does it.+ repoPath := t.TempDir()+ const n = 1000+ var due bool+ for i := 0; i < n; i++ {+ land := Land{Original: landOID(10000 + i), Landed: landOID(10000 + i)}+ compact, err := RecordLandIntents(repoPath, "refs/heads/main", invocationOf(land.Original), []Land{land})+ require.NoError(t, err)+ confirmLand(t, repoPath, "refs/heads/main", land.Original, land.Landed, invocationOf(land.Original))+ if compact {+ due = true+ require.NoError(t, CompactLandIntents(repoPath))+ }+ }++ require.True(t, due, "the append asked for a compaction")+ require.Less(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))+ })}func TestLandsIndexCompaction(t *testing.T) {diff --git a/internal/graft/changelog/log.go b/internal/graft/changelog/log.goindex 661cdb2..a1dee8b 100644--- a/internal/graft/changelog/log.go+++ b/internal/graft/changelog/log.go@@ -97,6 +97,11 @@ type Entry struct {// recorded on every land path and are never cut (see Land, WithLands and doc/graft/PLAN11.md).// They are empty for updates that are not lands.Lands []Land `json:"lands,omitempty"`+ // LandInvocation identifies the update that landed Lands: the invocation of its reference+ // transaction (see gitcmd.ChangelogMeta.Invocation), which its land intents carry too. A land+ // intent is confirmed, and may go, once the entry of its own update is durable. It is empty+ // without Lands.+ LandInvocation string `json:"land_invocation,omitempty"`}// entryFields is Entry without its JSON methods.@@ -175,6 +180,9 @@ type Meta struct {// LandReference is the branch that Lands landed on. Only the entry of the update of that// branch carries them (see LandsOf).LandReference string+ // LandInvocation identifies the update that recorded Lands, as its land intents do (see+ // Entry.LandInvocation).+ LandInvocation string// Lands are the identities of the changes that the updates landed, see Entry.Lands.Lands []Land}@@ -482,6 +490,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entriesMembers: normalizeMembers(meta.Members),MembersTruncated: meta.MembersTruncated,Lands: LandsOf(meta, update.Reference),+ LandInvocation: LandInvocationOf(meta, update.Reference),}line, err := json.Marshal(entry)diff --git a/internal/graft/corestore/sqlstore/changeset.go b/internal/graft/corestore/sqlstore/changeset.goindex c83cdd9..a1658a6 100644--- a/internal/graft/corestore/sqlstore/changeset.go+++ b/internal/graft/corestore/sqlstore/changeset.go@@ -256,6 +256,7 @@ func buildChangelogEntries(repo *repoState, meta *changelog.Meta, now time.Time,Members: slices.Clone(meta.Members),MembersTruncated: meta.MembersTruncated,Lands: changelog.LandsOf(*meta, u.Name),+ LandInvocation: changelog.LandInvocationOf(*meta, u.Name),})}