Skip to content

Git designed for AI agents.

PLAN11 re-review: exact intent confirmation, the empty payload, and fixes

Federico Elia53ea79716cac
diff --git a/internal/git/gitcmd/hooks_payload.go b/internal/git/gitcmd/hooks_payload.go
index e99ccc3..99394d1 100644
--- a/internal/git/gitcmd/hooks_payload.go
+++ b/internal/git/gitcmd/hooks_payload.go
@@ -264,7 +264,13 @@ func NewHooksPayload(
id, hasIdentity := identity.FromContext(ctx)
members, membersTruncated := capChangelogMembers(changelog.BatchFromContext(ctx))
- landReference, lands := changelog.LandsFromContext(ctx)
+ // The payload holds no branch for an update that lands nothing: an empty slice would not
+ // survive the encoding (omitempty), and the decoded payload would differ.
+ var landReference []byte
+ reference, lands := changelog.LandsFromContext(ctx)
+ if reference != "" {
+ landReference = []byte(reference)
+ }
return HooksPayload{
Repo: &gitalypb.Repository{
@@ -293,7 +299,7 @@ func NewHooksPayload(
Rewritten: changelog.IsRewritten(ctx),
Members: members,
MembersTruncated: membersTruncated,
- LandReference: []byte(landReference),
+ LandReference: landReference,
Lands: lands,
MVCCCommit: mvccCommitHandle(ctx, cfg),
},
diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.go
index ac91f9e..70d5c8d 100644
--- a/internal/gitaly/hook/referencetransaction.go
+++ b/internal/gitaly/hook/referencetransaction.go
@@ -226,7 +226,22 @@ func (m *GitLabHookManager) recordLandIntents(ctx context.Context, payload gitcm
return fmt.Errorf("resolving repository path: %w", err)
}
- return changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Lands)
+ compact, err := changelog.RecordLandIntents(repoPath, string(meta.LandReference), meta.Invocation, meta.Lands)
+ if err != nil {
+ return err
+ }
+
+ // The compaction reads the changelog and rewrites the intents: it runs after the hook, not
+ // under the locks of the references, and its failure leaves the intents as they are.
+ if compact {
+ go func() {
+ if err := changelog.CompactLandIntents(repoPath); err != nil {
+ m.logger.WithError(err).WithField("repository", repoPath).Error("graft: compacting the land intents failed")
+ }
+ }()
+ }
+
+ return nil
}
// prepareChangesets gives the prepared transaction its place in the order of the changeset log,
diff --git a/internal/gitaly/hook/referencetransaction_intents_test.go b/internal/gitaly/hook/referencetransaction_intents_test.go
new file mode 100644
index 0000000..155ebdd
--- /dev/null
+++ b/internal/gitaly/hook/referencetransaction_intents_test.go
@@ -0,0 +1,87 @@
+package hook
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/git/gitcmd"
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/internal/graft/changelog"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+)
+
+// TestReferenceTransactionHook_recordsLandIntents checks that the prepared phase of an update that
+// lands changes records their intents before git may commit it (doc/graft/PLAN11.md, 4.2), that an
+// update of another reference records none, and that a failure to record them aborts the update.
+func TestReferenceTransactionHook_recordsLandIntents(t *testing.T) {
+ t.Parallel()
+
+ land := changelog.Land{Original: strings.Repeat("a", 40), Landed: strings.Repeat("b", 40)}
+
+ // prepare runs the prepared phase of an update of reference whose payload lands land on main,
+ // in a new repository, and returns the repository's path and the error of the hook.
+ prepare := func(t *testing.T, cfg config.Cfg, reference string, setup func(repoPath string)) (string, error) {
+ ctx := testhelper.Context(t)
+ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ repoPath, err := config.NewLocator(cfg).GetRepoPath(ctx, repo)
+ require.NoError(t, err)
+ if setup != nil {
+ setup(repoPath)
+ }
+
+ landCtx, err := changelog.WithLands(ctx, "refs/heads/main", []changelog.Land{land})
+ require.NoError(t, err)
+ env, err := gitcmd.NewHooksPayload(landCtx, cfg, repo, gittest.DefaultObjectHash, nil, nil, gitcmd.ReferenceTransactionHook, nil).Env()
+ require.NoError(t, err)
+
+ changes := fmt.Sprintf("%s %s %s\n", gittest.DefaultObjectHash.EmptyTreeOID, land.Landed, reference)
+ return repoPath, newMembersTestManager(t, cfg).ReferenceTransactionHook(context.Background(), ReferenceTransactionPrepared, []string{env}, strings.NewReader(changes))
+ }
+
+ t.Run("an update of the branch of the lands", func(t *testing.T) {
+ t.Parallel()
+
+ repoPath, err := prepare(t, testcfg.Build(t), "refs/heads/main", nil)
+ require.NoError(t, err)
+
+ records, err := changelog.LookupLand(repoPath, "refs/heads/main", land.Original)
+ require.NoError(t, err)
+ require.Equal(t, []changelog.LandRecord{{Landed: land.Landed}}, records)
+ })
+
+ t.Run("an update of another reference", func(t *testing.T) {
+ t.Parallel()
+
+ repoPath, err := prepare(t, testcfg.Build(t), "refs/heads/other", nil)
+ require.NoError(t, err)
+ require.NoFileExists(t, filepath.Join(repoPath, "graft", "changelog.intents"))
+ })
+
+ t.Run("without a changelog", func(t *testing.T) {
+ t.Parallel()
+
+ cfg := testcfg.Build(t)
+ cfg.Graft.Changelog.Disabled = true
+ repoPath, err := prepare(t, cfg, "refs/heads/main", nil)
+ require.NoError(t, err)
+ require.NoFileExists(t, filepath.Join(repoPath, "graft", "changelog.intents"))
+ })
+
+ t.Run("a failure to record the intents aborts the update", func(t *testing.T) {
+ t.Parallel()
+
+ // The intents file cannot be opened for writing: a directory is in its place.
+ _, err := prepare(t, testcfg.Build(t), "refs/heads/main", func(repoPath string) {
+ require.NoError(t, os.MkdirAll(filepath.Join(repoPath, "graft", "changelog.intents"), 0o755))
+ })
+ require.ErrorContains(t, err, "recording the land intents")
+ })
+}
diff --git a/internal/gitaly/hook/storage_voter.go b/internal/gitaly/hook/storage_voter.go
index a554395..b07e933 100644
--- a/internal/gitaly/hook/storage_voter.go
+++ b/internal/gitaly/hook/storage_voter.go
@@ -105,6 +105,9 @@ func changelogMetaOf(payload gitcmd.HooksPayload, graftMeta gitcmd.ChangelogMeta
LandReference: string(graftMeta.LandReference),
Lands: graftMeta.Lands,
}
+ if len(graftMeta.Lands) > 0 {
+ meta.LandInvocation = graftMeta.Invocation
+ }
if meta.Operation == "" {
meta.Operation = changelog.UnknownOperation
}
diff --git a/internal/gitaly/service/operations/land_identity.go b/internal/gitaly/service/operations/land_identity.go
index 6b31413..79d9316 100644
--- a/internal/gitaly/service/operations/land_identity.go
+++ b/internal/gitaly/service/operations/land_identity.go
@@ -85,14 +85,16 @@ func (a *landingAttempt) landedEarlier(ctx context.Context, reference git.Refere
onBranch, err := a.repo.IsAncestor(ctx, landed.Revision(), tip.Revision())
if err != nil {
- // The commit does not exist: an intent of an update that was not committed, whose
- // objects were never migrated, or a land whose branch was reset past it and whose
- // commit was pruned since. Either way the change is not on the branch through it.
+ // The commit does not exist: it was pruned since, after the branch was reset past it,
+ // or its update was aborted and its objects pruned. The change is not on the branch
+ // through it.
if errors.As(err, new(localrepo.InvalidCommitError)) {
continue
}
return nil, structerr.NewInternal("checking the earlier land of %s: %w", commitID, err)
}
+ // An update migrates its objects before it is prepared, so the commit of an intent whose
+ // update was aborted exists, off the branch, and does not answer.
if !onBranch {
continue
}
diff --git a/internal/gitaly/service/operations/land_identity_test.go b/internal/gitaly/service/operations/land_identity_test.go
index 68039a5..02951f8 100644
--- a/internal/gitaly/service/operations/land_identity_test.go
+++ b/internal/gitaly/service/operations/land_identity_test.go
@@ -7,6 +7,7 @@ import (
"os"
"strconv"
"strings"
+ "sync/atomic"
"testing"
"github.com/stretchr/testify/require"
@@ -15,8 +16,13 @@ import (
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gittest"
"github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/internal/gitaly/transaction"
"github.com/feder-cr/git-ai/internal/graft/changelog"
+ "github.com/feder-cr/git-ai/internal/grpc/metadata"
"github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testserver"
+ "github.com/feder-cr/git-ai/internal/transaction/txinfo"
+ "github.com/feder-cr/git-ai/internal/transaction/voting"
)
// TestLandBatch_retryAfterALostResponse lands two changes by rebase in one batch, and then lands
@@ -113,9 +119,9 @@ func testUserLandIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.C
requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, landed)
}
-// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing records intents whose updates git never
-// committed, as an update that was aborted after its prepared phase leaves them: their landed
-// commit is not on the branch, so the change lands.
+// TestUserLand_intentOfAnUncommittedUpdateAnswersNothing aborts a land after the prepared phase of
+// its update, which leaves its intent: the intent's landed commit exists (the objects of an update
+// are migrated before it is prepared) but is not on the branch, so the next land lands the change.
func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {
t.Parallel()
@@ -125,19 +131,46 @@ func TestUserLand_intentOfAnUncommittedUpdateAnswersNothing(t *testing.T) {
func testUserLandIntentOfAnUncommittedUpdateAnswersNothing(t *testing.T, ctx context.Context) {
t.Parallel()
- ctx, cfg, client := setupOperationsService(t, ctx)
+ txManager := transaction.MockManager{}
+ ctx, cfg, client := setupOperationsService(t, ctx, testserver.WithTransactionManager(&txManager))
+
+ ctx, err := txinfo.InjectTransaction(ctx, 1, "node", true)
+ require.NoError(t, err)
+ ctx = metadata.IncomingToOutgoing(ctx)
+
+ // The repository is set up without the transaction failing.
+ txManager.VoteFn = func(context.Context, txinfo.Transaction, voting.Vote, voting.Phase) error { return nil }
fx := newRetryFixture(t, ctx, cfg)
- // One intent names a commit that does not exist, the other one that is not on the branch.
- require.NoError(t, changelog.RecordLandIntents(fx.repoPath, git.DefaultRef.String(), []changelog.Land{
- {Original: fx.change.String(), Landed: strings.Repeat("1", len(fx.change))},
- {Original: fx.change.String(), Landed: fx.change.String()},
- }))
+ // The first land fails its vote in the prepared phase of its update, after the hook recorded
+ // its intent: git aborts the update. Its objects were migrated before the update, so its
+ // landed commit exists, off the branch.
+ var failed atomic.Bool
+ txManager.VoteFn = func(_ context.Context, _ txinfo.Transaction, _ voting.Vote, phase voting.Phase) error {
+ if phase == voting.Prepared && failed.CompareAndSwap(false, true) {
+ return errors.New("the transaction lost its quorum")
+ }
+ return nil
+ }
+
+ request := landRequest(fx.repoProto, git.DefaultBranch, fx.change)
+ _, err = client.UserLand(ctx, request)
+ require.Error(t, err)
+ require.True(t, failed.Load())
+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, fx.tip)
- resp, err := client.UserLand(ctx, landRequest(fx.repoProto, git.DefaultBranch, fx.change))
+ records, err := changelog.LookupLand(fx.repoPath, git.DefaultRef.String(), fx.change.String())
require.NoError(t, err)
- require.False(t, resp.GetAlreadyLanded())
- require.True(t, resp.GetRewritten())
+ require.Len(t, records, 1, "the aborted update left its intent")
+ aborted := records[0].Landed
+ require.Equal(t, "commit\n", string(gittest.Exec(t, cfg, "-C", fx.repoPath, "cat-file", "-t", aborted)), "the aborted commit exists")
+
+ // The intent's commit is not on the branch, so the land lands the change.
+ second, err := client.UserLand(ctx, request)
+ require.NoError(t, err)
+ require.False(t, second.GetAlreadyLanded())
+ require.True(t, second.GetRewritten())
+ requireBranchAt(t, cfg, fx.repoPath, git.DefaultBranch, git.ObjectID(second.GetLandedCommitId()))
}
// TestUserLand_identityOfABranchWhoseNameIsNotUTF8 lands a change by rebase on a branch whose name
@@ -247,6 +280,58 @@ func TestUserLand_branchNameTooLongForTheIdentity(t *testing.T) {
_, err := client.UserLand(ctx, landRequest(fx.repoProto, branch, fx.change))
testhelper.RequireGrpcCode(t, err, codes.InvalidArgument)
require.ErrorContains(t, err, "is too long to record the identity of a land on it")
+ require.Less(t, len(err.Error()), 512, "the error quotes the beginning of the name, not all of it")
+}
+
+// TestLandBatch_intentAnswersWhatTheChangelogMisses lands two changes in one batch, loses the
+// changelog entry of their update, and lands them again: the intents the hook recorded for the
+// batch's update answer.
+func TestLandBatch_intentAnswersWhatTheChangelogMisses(t *testing.T) {
+ t.Parallel()
+
+ runLandBatchTest(t, testLandBatchIntentAnswersWhatTheChangelogMisses)
+}
+
+func testLandBatchIntentAnswersWhatTheChangelogMisses(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ ctx, cfg, client := setupAgentLandService(t, ctx, agentBatchingGraft(nil))
+ repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg)
+
+ baseEntry := gittest.TreeEntry{Mode: "100644", Path: "base", Content: "base"}
+ base := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch), gittest.WithTreeEntries(baseEntry))
+ gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch), gittest.WithParents(base),
+ gittest.WithTreeEntries(baseEntry, gittest.TreeEntry{Mode: "100644", Path: "moved", Content: "moved"}))
+ change := func(name string) git.ObjectID {
+ return gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(base),
+ gittest.WithTreeEntries(baseEntry, gittest.TreeEntry{Mode: "100644", Path: name, Content: name}))
+ }
+ a, b := change("a"), change("b")
+
+ first, errs := landInOrder(ctx, client, repoProto,
+ agentLanding{agent: "agent-a", commit: a},
+ agentLanding{agent: "agent-b", commit: b},
+ )
+ for i := range errs {
+ require.NoError(t, errs[i])
+ require.Equal(t, int32(2), first[i].GetBatchSize(), "both changes land in one batch")
+ }
+ tip := gittest.ResolveRevision(t, cfg, repoPath, git.DefaultRef.String())
+
+ // The changelog lost the entry of the batch's update, its only entry.
+ require.NoError(t, os.Remove(changelog.Path(repoPath)))
+
+ second, errs := landInOrder(ctx, client, repoProto,
+ agentLanding{agent: "agent-a", commit: a},
+ agentLanding{agent: "agent-b", commit: b},
+ )
+ for i := range errs {
+ require.NoError(t, errs[i])
+ require.True(t, second[i].GetAlreadyLanded())
+ require.Equal(t, first[i].GetLandedCommitId(), second[i].GetLandedCommitId())
+ require.Zero(t, second[i].GetLandedSequence())
+ }
+ requireBranchAt(t, cfg, repoPath, git.DefaultBranch, tip)
}
// TestLandBatchExecutorRetryInTheBatchOfItsFirstLand puts a request and its retry in one batch:
diff --git a/internal/graft/changelog/lands.go b/internal/graft/changelog/lands.go
index c677397..1f54f7e 100644
--- a/internal/graft/changelog/lands.go
+++ b/internal/graft/changelog/lands.go
@@ -70,7 +70,8 @@ var fullBatchLandsSize = func() int {
// too long for them cannot be landed on, since a land would not record its identity.
func CheckLandReference(reference string) error {
if size := fullBatchLandsSize + base64.StdEncoding.EncodedLen(len(reference)); size > MaxLandsSize {
- return fmt.Errorf("the name of branch %q is too long to record the identity of a land on it: %d bytes, at most %d",
+ // The name is too long to be quoted whole: its first bytes say which branch it is.
+ return fmt.Errorf("the name of branch %.64q... is too long to record the identity of a land on it: %d bytes, at most %d",
reference, len(reference), (MaxLandsSize-fullBatchLandsSize)/4*3)
}
return nil
@@ -163,3 +164,13 @@ func LandsOf(meta Meta, reference string) []Land {
return append([]Land(nil), meta.Lands...)
}
+
+// LandInvocationOf returns the invocation that the changelog entry of an update of reference
+// carries under meta with its lands, see Entry.LandInvocation: empty when it carries no lands.
+func LandInvocationOf(meta Meta, reference string) string {
+ if len(LandsOf(meta, reference)) == 0 {
+ return ""
+ }
+
+ return meta.LandInvocation
+}
diff --git a/internal/graft/changelog/lands_index.go b/internal/graft/changelog/lands_index.go
index 222e12a..45bcd4b 100644
--- a/internal/graft/changelog/lands_index.go
+++ b/internal/graft/changelog/lands_index.go
@@ -101,7 +101,10 @@ type landsIndex struct {
// logDevice and logInode identify the log file that records was read from.
logDevice, logInode uint64
// records maps the lands of the log entries up to mark to their last record.
- records map[landsIndexKey]LandRecord
+ records map[landsIndexKey]indexRecord
+ // confirmed holds the invocations of the land updates whose changelog entries the index read
+ // (Entry.LandInvocation): their intents may go. An invocation leaves it with its intents.
+ confirmed map[string]struct{}
// recordLines is the number of record lines of the index file, superseded ones included.
recordLines int
// fileSize is the size of the index file when this process last read or wrote it. A file of
@@ -112,10 +115,21 @@ type landsIndex struct {
// known of that file.
intents map[landsIndexKey][]landIntent
intentsFile intentsFile
- // intentsKept is the size of the intents file after its last compaction by this process. The
- // file is compacted again once it doubled, so that intents that cannot go yet are not
- // rewritten by every lookup.
- intentsKept int64
+}
+
+// indexRecord is the last record of a land in an index.
+type indexRecord struct {
+ LandRecord
+ // invocation identifies the update of the record, see Entry.LandInvocation.
+ invocation string
+}
+
+// forget drops what the index knows, after a catch-up that failed: the next lookup reads the
+// index and the intents again. The caller must hold i.mu.
+func (i *landsIndex) forget() {
+ i.loaded = false
+ i.intents = nil
+ i.intentsFile = intentsFile{}
}
var (
@@ -194,10 +208,12 @@ func landsPath(repoPath string) string {
// LookupLand returns what the repository whose git directory is repoPath knows of the lands of
// original on reference (doc/graft/PLAN11.md): first the last land that the changelog recorded,
// with the sequence of its entry and the commit it landed as, then the land intents (see
-// landIntent) of lands that the changelog does not record, with a zero sequence. A land answers
-// only when its landed commit is on the branch, which the caller checks. The result is empty when
-// nothing is known. The lookup reads the durable entries of the log (see AppendGroup) and the
-// intents that the index does not cover yet, and adds them to the index first.
+// landIntent) of other landed commits, with a zero sequence. A land answers only when its landed
+// commit is on the branch, which the caller checks: an intent whose commit is on the branch proves
+// the land by itself, whatever the changelog recorded since, so no intent is left out but those of
+// the recorded commit. The result is empty when nothing is known. The lookup reads the durable
+// entries of the log (see AppendGroup) and the intents that the index does not cover yet, and adds
+// them to the index first.
func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {
index, release := acquireLandsIndex(landsPath(repoPath))
defer release()
@@ -207,21 +223,22 @@ func LookupLand(repoPath, reference, original string) ([]LandRecord, error) {
if err := index.catchUp(repoPath); err != nil {
// What the failed catch-up did to the index is not known, so the next lookup reads it again.
- index.loaded = false
- index.intentsFile = intentsFile{}
+ index.forget()
return nil, err
}
key := landsIndexKey{reference: reference, original: original}
var lands []LandRecord
- confirmed, found := index.records[key]
+ recorded, found := index.records[key]
if found {
- lands = append(lands, confirmed)
+ lands = append(lands, recorded.LandRecord)
}
+ seen := map[string]bool{recorded.Landed: found}
for _, intent := range index.intents[key] {
- if found && (intent.Landed == confirmed.Landed || confirmed.Sequence > intent.Head) {
+ if seen[intent.Landed] {
continue
}
+ seen[intent.Landed] = true
lands = append(lands, LandRecord{Landed: intent.Landed})
}
@@ -248,7 +265,10 @@ func (i *landsIndex) estimate() int64 {
const overhead = 96
var bytes int64
for key, record := range i.records {
- bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)) + overhead
+ bytes += int64(len(key.reference)+len(key.original)+len(record.Landed)+len(record.invocation)) + overhead
+ }
+ for invocation := range i.confirmed {
+ bytes += int64(len(invocation)) + overhead
}
for key, intents := range i.intents {
for _, intent := range intents {
@@ -333,7 +353,7 @@ func (i *landsIndex) catchUpLog(repoPath string) error {
}
for _, record := range block.records {
- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}
+ i.add(record)
}
i.recordLines += len(block.records)
i.mark = block.mark
@@ -356,7 +376,7 @@ func (i *landsIndex) catchUpLog(repoPath string) error {
func (i *landsIndex) compact(path string) error {
block := landsBlock{mark: i.mark}
for key, record := range i.records {
- block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed})
+ block.records = append(block.records, landsBlockRecord{sequence: record.Sequence, reference: key.reference, original: key.original, landed: record.Landed, invocation: record.invocation})
}
sort.Slice(block.records, func(a, b int) bool { return block.records[a].sequence < block.records[b].sequence })
@@ -398,7 +418,7 @@ func (i *landsIndex) catchUpIntents(repoPath string) error {
}
if reset {
// The file is not the one that was read before, and the intents read before are not
- // its: they are dropped, and the file is read again from its start.
+ // its: they are dropped, and the file is read from its start.
i.intents = map[landsIndexKey][]landIntent{}
if known, _, err = readIntents(path, intentsFile{}, add); err != nil {
return err
@@ -406,40 +426,72 @@ func (i *landsIndex) catchUpIntents(repoPath string) error {
}
i.intentsFile = known
- if known.offset < compactIntentsMinSize || known.offset < 2*i.intentsKept {
+ // An intent may go once the changelog entry of its own update is durable. The file is
+ // compacted from compactIntentsMinSize on, when at least half of its intents may go, so that
+ // intents that must stay are not rewritten again and again.
+ confirmed := func(intent landIntent) bool {
+ _, ok := i.confirmed[intent.Invocation]
+ return intent.Invocation != "" && ok
+ }
+ total, droppable := 0, 0
+ for _, intents := range i.intents {
+ for _, intent := range intents {
+ total++
+ if confirmed(intent) {
+ droppable++
+ }
+ }
+ }
+ if known.offset < compactIntentsMinSize || 2*droppable < total {
return nil
}
- // An intent may go once the changelog confirms it (a record of its land with its landed
- // commit) or supersedes it (a later record of the same original commit on the branch).
- if err := compactIntents(repoPath, path, func(intent landIntent) bool {
- record, found := i.records[landsIndexKey{reference: string(intent.Reference), original: intent.Original}]
- return found && (record.Landed == intent.Landed || record.Sequence > intent.Head)
- }); err != nil {
+ if err := compactIntents(repoPath, path, confirmed); err != nil {
return err
}
+ // The invocations of the dropped intents are not needed any more.
+ for _, intents := range i.intents {
+ for _, intent := range intents {
+ if confirmed(intent) {
+ delete(i.confirmed, intent.Invocation)
+ }
+ }
+ }
+
i.intents = map[landsIndexKey][]landIntent{}
known, _, err = readIntents(path, intentsFile{}, add)
if err != nil {
return err
}
i.intentsFile = known
- i.intentsKept = known.offset
return nil
}
+// add records a land in the index: as the last record of its original commit on its branch, and
+// its invocation as confirmed. The caller must hold i.mu.
+func (i *landsIndex) add(record landsBlockRecord) {
+ i.records[landsIndexKey{reference: record.reference, original: record.original}] = indexRecord{
+ LandRecord: LandRecord{Sequence: record.sequence, Landed: record.landed},
+ invocation: record.invocation,
+ }
+ if record.invocation != "" {
+ i.confirmed[record.invocation] = struct{}{}
+ }
+}
+
// errLandsIndexStale is returned by scanLog when the log entry at the mark of the index is not the
// one the mark names.
var errLandsIndexStale = errors.New("the land index does not describe the log")
// landsBlockRecord is a record of a block, see landsBlock.
type landsBlockRecord struct {
- sequence uint64
- reference string
- original string
- landed string
+ sequence uint64
+ reference string
+ original string
+ landed string
+ invocation string
}
// landsBlock is the records of the lands of the log entries after the mark of an index, and the
@@ -472,10 +524,11 @@ func (i *landsIndex) scanLog(repoPath string) (landsBlock, error) {
for _, land := range entry.Lands {
block.records = append(block.records, landsBlockRecord{
- sequence: entry.Sequence,
- reference: entry.Reference,
- original: land.Original,
- landed: land.Landed,
+ sequence: entry.Sequence,
+ reference: entry.Reference,
+ original: land.Original,
+ landed: land.Landed,
+ invocation: entry.LandInvocation,
})
}
block.mark = landsMark{sequence: entry.Sequence, reference: entry.Reference, newOID: entry.NewOID}
@@ -500,6 +553,8 @@ type landsLine struct {
Reference []byte `json:"ref"`
Original string `json:"original,omitempty"`
Landed string `json:"landed,omitempty"`
+ // Invocation identifies the update of the record, see Entry.LandInvocation.
+ Invocation string `json:"inv,omitempty"`
// Mark is the sequence of the log entry that the mark names; Reference and NewOID are its
// reference and new value, CRC the CRC-32 (IEEE) of the record lines of the block.
Mark uint64 `json:"mark,omitempty"`
@@ -512,10 +567,11 @@ func (b landsBlock) encode() ([]byte, error) {
var buf bytes.Buffer
for _, record := range b.records {
line, err := json.Marshal(landsLine{
- Sequence: record.sequence,
- Reference: []byte(record.reference),
- Original: record.original,
- Landed: record.landed,
+ Sequence: record.sequence,
+ Reference: []byte(record.reference),
+ Original: record.original,
+ Landed: record.landed,
+ Invocation: record.invocation,
})
if err != nil {
return nil, fmt.Errorf("encoding land index record: %w", err)
@@ -542,7 +598,8 @@ func (b landsBlock) encode() ([]byte, error) {
// reset empties the in-memory index of a file of the given size.
func (i *landsIndex) reset(fileSize int64) {
i.mark = landsMark{}
- i.records = map[landsIndexKey]LandRecord{}
+ i.records = map[landsIndexKey]indexRecord{}
+ i.confirmed = map[string]struct{}{}
i.recordLines = 0
i.fileSize = fileSize
i.loaded = false
@@ -585,10 +642,11 @@ func (i *landsIndex) load(f *os.File) error {
break
}
pending = append(pending, landsBlockRecord{
- sequence: decoded.Sequence,
- reference: string(decoded.Reference),
- original: decoded.Original,
- landed: decoded.Landed,
+ sequence: decoded.Sequence,
+ reference: string(decoded.Reference),
+ original: decoded.Original,
+ landed: decoded.Landed,
+ invocation: decoded.Invocation,
})
crc.Write(line)
continue
@@ -600,7 +658,7 @@ func (i *landsIndex) load(f *os.File) error {
}
for _, record := range pending {
- i.records[landsIndexKey{reference: record.reference, original: record.original}] = LandRecord{Sequence: record.sequence, Landed: record.landed}
+ i.add(record)
}
i.recordLines += len(pending)
i.mark = landsMark{sequence: decoded.Mark, reference: string(decoded.Reference), newOID: decoded.NewOID}
diff --git a/internal/graft/changelog/lands_intents.go b/internal/graft/changelog/lands_intents.go
index 772b8f8..e623ab8 100644
--- a/internal/graft/changelog/lands_intents.go
+++ b/internal/graft/changelog/lands_intents.go
@@ -28,10 +28,10 @@ const intentsFileName = "changelog.intents"
// committed answers nothing.
//
// The file is JSON lines, appended under the lock file of the intents and fsync'd before the
-// prepared phase ends. Unlike the index it is not derived data. An intent is dropped by a
-// compaction once the changelog confirms it (a record of its land with its landed commit) or
-// supersedes it (a later record of the same original commit on the branch); the intent of an
-// update that was never committed and never landed again stays.
+// prepared phase ends. Unlike the index it is not derived data. An intent carries the invocation
+// of its update, and so does the changelog entry that the update writes once it is committed
+// (Entry.LandInvocation): a compaction drops an intent once the entry of its own update is
+// durable, and only then. The intent of an update that was never committed stays.
// landIntent is one line of the intents file.
type landIntent struct {
@@ -40,10 +40,8 @@ type landIntent struct {
Reference []byte `json:"ref"`
Original string `json:"original"`
Landed string `json:"landed"`
- // Head is the sequence of the last durable changelog entry when the intent was recorded. The
- // entry of the land, if any, comes after it, so a record of the same commit at a later
- // sequence confirms or supersedes the intent.
- Head uint64 `json:"head"`
+ // Invocation identifies the update that recorded the intent, see Entry.LandInvocation.
+ Invocation string `json:"invocation"`
}
// intentsPath returns the path of the intents file of the repository whose git directory is
@@ -52,75 +50,145 @@ func intentsPath(repoPath string) string {
return filepath.Join(repoPath, dirName, intentsFileName)
}
-// intentDirsSynced holds the intents files of this process whose directory entries are durable,
-// see RecordLandIntents.
-var intentDirsSynced sync.Map
+// intentsFileKey identifies an intents file: its path, and the file at that path.
+type intentsFileKey struct {
+ path string
+ device, inode uint64
+}
-// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the
-// update of reference about to be committed lands lands (see the comment above landIntent). It
-// returns once the intents are fsync'd; an error must abort the update, which would otherwise be
-// committed without its identity.
-func RecordLandIntents(repoPath, reference string, lands []Land) error {
- head, err := readLogHead(repoPath)
- if err != nil {
- return err
- }
+var (
+ // intentDirsSynced holds the intents files of this process whose directory entries are
+ // durable, see RecordLandIntents. A file created again at the same path, as with a
+ // repository created again, is another file.
+ intentDirsSynced sync.Map
+ // intentsCompactionSize holds, by path, the size of the intents file when RecordLandIntents
+ // last asked for a compaction, see compactionDue.
+ intentsCompactionSize sync.Map
+)
+// RecordLandIntents records durably, in the repository whose git directory is repoPath, that the
+// update of reference about to be committed lands lands, as the update identified by invocation
+// (see the comment above landIntent). It returns once the intents are fsync'd; an error must abort
+// the update, which would otherwise be committed without its identity. compact reports that the
+// file grew so that its compaction is due: the caller runs CompactLandIntents, outside the locks
+// of the update.
+func RecordLandIntents(repoPath, reference, invocation string, lands []Land) (compact bool, err error) {
path := intentsPath(repoPath)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
- return fmt.Errorf("creating changelog directory: %w", err)
+ return false, fmt.Errorf("creating changelog directory: %w", err)
}
unlock, err := lockFile(path + lockSuffix)
if err != nil {
- return err
+ return false, err
}
defer unlock()
+ _, statErr := os.Stat(path)
+ created := errors.Is(statErr, fs.ErrNotExist)
+
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
- return fmt.Errorf("opening land intents: %w", err)
+ return false, fmt.Errorf("opening land intents: %w", err)
}
defer f.Close()
+ info, err := f.Stat()
+ if err != nil {
+ return false, fmt.Errorf("stat land intents: %w", err)
+ }
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return false, fmt.Errorf("stat land intents: no file identity in %T", info.Sys())
+ }
+
// The bytes after the last newline are a torn line of a writer that crashed before its fsync,
// whose update was therefore never committed. They are cut off, or the next line would be
// joined to them.
end, err := endOfLastLine(f)
if err != nil {
- return err
+ return false, err
}
var buf bytes.Buffer
for _, land := range lands {
- line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Head: head.sequence})
+ line, err := json.Marshal(landIntent{Reference: []byte(reference), Original: land.Original, Landed: land.Landed, Invocation: invocation})
if err != nil {
- return fmt.Errorf("encoding land intent: %w", err)
+ return false, fmt.Errorf("encoding land intent: %w", err)
}
buf.Write(line)
buf.WriteByte('\n')
}
if err := truncateFile(f, end); err != nil {
- return fmt.Errorf("truncating land intents: %w", err)
+ return false, fmt.Errorf("truncating land intents: %w", err)
}
if _, err := f.WriteAt(buf.Bytes(), end); err != nil {
- return fmt.Errorf("writing land intents: %w", err)
+ return false, fmt.Errorf("writing land intents: %w", err)
}
if err := syncFile(f); err != nil {
- return fmt.Errorf("syncing land intents: %w", err)
+ return false, fmt.Errorf("syncing land intents: %w", err)
}
- // The intents are durable only once the directory entries of their file are, which a writer
- // that created the file may have crashed before syncing: the first write of this process
- // syncs them.
- if _, synced := intentDirsSynced.Load(path); !synced {
+ // The intents are durable only once the directory entries of their file are: when this write
+ // created the file, and on the first write of this process to the file, which a writer that
+ // created it may have crashed before syncing.
+ key := intentsFileKey{path: path, device: uint64(stat.Dev), inode: stat.Ino}
+ if _, synced := intentDirsSynced.Load(key); created || !synced {
if err := syncLogDirs(repoPath, path); err != nil {
- return err
+ return false, err
}
- intentDirsSynced.Store(path, struct{}{})
+ intentDirsSynced.Store(key, struct{}{})
}
+ return compactionDue(path, end+int64(buf.Len())), nil
+}
+
+// compactionDue reports whether an intents file of the given size is due for a compaction: from
+// compactIntentsMinSize on, each time it doubled since the last time it was due, or since the last
+// CompactLandIntents left it, in this process.
+func compactionDue(path string, size int64) bool {
+ if size < compactIntentsMinSize {
+ return false
+ }
+
+ last, _ := intentsCompactionSize.Load(path)
+ if lastSize, ok := last.(int64); ok && size < 2*lastSize {
+ return false
+ }
+
+ intentsCompactionSize.Store(path, size)
+ return true
+}
+
+// compactionsRunning holds the repositories whose intents CompactLandIntents compacts right now.
+var compactionsRunning sync.Map
+
+// CompactLandIntents brings the land index of the repository whose git directory is repoPath up to
+// its changelog and intents, which compacts the intents file when enough of its intents are
+// confirmed (see landsIndex.catchUpIntents). RecordLandIntents asks for it, so that the intents of
+// a repository whose lands never look up an earlier land (fast-forwards only) are compacted too. A
+// call while another one runs for the repository returns at once.
+func CompactLandIntents(repoPath string) error {
+ if _, running := compactionsRunning.LoadOrStore(repoPath, struct{}{}); running {
+ return nil
+ }
+ defer compactionsRunning.Delete(repoPath)
+
+ index, release := acquireLandsIndex(landsPath(repoPath))
+ defer release()
+
+ index.mu.Lock()
+ defer index.mu.Unlock()
+
+ if err := index.catchUp(repoPath); err != nil {
+ index.forget()
+ return err
+ }
+
+ // The next compaction is due once the file doubled from what this one left.
+ intentsCompactionSize.Store(intentsPath(repoPath), index.intentsFile.offset)
+
return nil
}
@@ -148,10 +216,11 @@ type intentsFile struct {
}
// readIntents reads the complete lines of the intents file at path after known, calling add for
-// each intent. It returns what the reader knows of the file after the read. When the file is not
-// the one known (it was replaced) or is shorter (it was replaced or cut), it is read from the
-// start and reset is true: the intents read before do not describe it any more. A missing file has
-// no intents.
+// each intent, and returns what the reader knows of the file after the read. A reader that knows
+// nothing of the file (the zero known) reads it from its start. When the file is not the one known
+// (it was replaced) or is shorter (it was replaced or cut), nothing is read and reset is true: the
+// intents read before do not describe it, and the caller drops them and reads it again knowing
+// nothing. A missing file has no intents.
func readIntents(path string, known intentsFile, add func(landIntent)) (intentsFile, bool, error) {
f, err := os.Open(path)
if err != nil {
@@ -172,9 +241,8 @@ func readIntents(path string, known intentsFile, add func(landIntent)) (intentsF
}
current := intentsFile{device: uint64(stat.Dev), inode: stat.Ino, offset: known.offset}
- reset := current.device != known.device || current.inode != known.inode || info.Size() < known.offset
- if reset {
- current.offset = 0
+ if known != (intentsFile{}) && (current.device != known.device || current.inode != known.inode || info.Size() < known.offset) {
+ return intentsFile{}, true, nil
}
r := bufio.NewReader(io.NewSectionReader(f, current.offset, info.Size()-current.offset))
@@ -199,7 +267,7 @@ func readIntents(path string, known intentsFile, add func(landIntent)) (intentsF
add(intent)
}
- return current, reset, nil
+ return current, false, nil
}
// compactIntents rewrites the intents file at path without the intents that drop says may go,
diff --git a/internal/graft/changelog/lands_intents_test.go b/internal/graft/changelog/lands_intents_test.go
index f42268a..8cdbfa3 100644
--- a/internal/graft/changelog/lands_intents_test.go
+++ b/internal/graft/changelog/lands_intents_test.go
@@ -12,10 +12,26 @@ import (
"github.com/stretchr/testify/require"
)
-// recordIntent records the intent of a land of original on reference as landed.
+// invocationOf returns the invocation that the tests give the update of the land of original.
+func invocationOf(original string) string { return "inv-" + original }
+
+// recordIntent records the intent of a land of original on reference as landed, by the update
+// invocationOf(original).
func recordIntent(t *testing.T, repoPath, reference, original, landed string) {
t.Helper()
- require.NoError(t, RecordLandIntents(repoPath, reference, []Land{{Original: original, Landed: landed}}))
+ _, err := RecordLandIntents(repoPath, reference, invocationOf(original), []Land{{Original: original, Landed: landed}})
+ require.NoError(t, err)
+}
+
+// confirmLand records the changelog entry of the update invocation, which landed original as
+// landed on reference.
+func confirmLand(t *testing.T, repoPath, reference, original, landed, invocation string) uint64 {
+ t.Helper()
+
+ entries, err := Append(repoPath, Meta{Operation: "land", LandReference: reference, LandInvocation: invocation, Lands: []Land{{Original: original, Landed: landed}}},
+ []Update{{Reference: reference, OldOID: landOID(0), NewOID: landed}})
+ require.NoError(t, err)
+ return entries[0].Sequence
}
// intentsOf returns the intents in the intents file of the repository.
@@ -43,26 +59,28 @@ func TestLookupLandAnswersFromIntents(t *testing.T) {
require.Empty(t, lookupAll(t, repoPath, "refs/heads/other", landOID(1)))
})
- t.Run("an intent that the changelog confirms", func(t *testing.T) {
+ t.Run("an intent of the recorded commit", func(t *testing.T) {
repoPath := t.TempDir()
recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
- seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ seq := confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), invocationOf(landOID(1)))
require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
})
- t.Run("an intent that a later land supersedes", func(t *testing.T) {
- // The update of the intent was not committed, and the change was landed again.
+ t.Run("an intent of another commit than the recorded one", func(t *testing.T) {
+ // Review m1: land A of X is committed and its entry is pending; the branch is reset; land
+ // B of X is committed with its intent; A's entry is written, and B's is lost. The intent
+ // of B is not left out because a record came after it: it proves B by itself, when its
+ // commit is on the branch.
repoPath := t.TempDir()
- recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
- seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(3))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))
+ seq := confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), "inv-a")
- require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(3)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
+ require.Equal(t, []LandRecord{{Sequence: seq, Landed: landOID(2)}, {Landed: landOID(3)}},
+ lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
})
t.Run("an intent after the recorded land", func(t *testing.T) {
- // The change landed, the branch was reset past it, and its new land was committed
- // without the changelog recording it: both are known, the record first.
repoPath := t.TempDir()
seq := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(3))
@@ -87,6 +105,17 @@ func TestLookupLandAnswersFromIntents(t *testing.T) {
recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))
require.Equal(t, []LandRecord{{Landed: landOID(4)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(3)))
})
+
+ t.Run("the entry carries the invocation of its update", func(t *testing.T) {
+ repoPath := t.TempDir()
+ confirmLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2), "inv-x")
+ appendPlain(t, repoPath, "refs/heads/main", landOID(9))
+
+ entries, err := List(repoPath, 0, 10)
+ require.NoError(t, err)
+ require.Equal(t, "inv-x", entries[0].LandInvocation)
+ require.Empty(t, entries[1].LandInvocation, "an entry without lands has no invocation")
+ })
}
func TestLandIntentsFile(t *testing.T) {
@@ -117,18 +146,30 @@ func TestLandIntentsFile(t *testing.T) {
require.Equal(t, []LandRecord{{Landed: landOID(2)}}, lookupAll(t, repoPath, "refs/heads/main", landOID(1)))
})
- t.Run("an intent records the head of the log", func(t *testing.T) {
+ t.Run("an intent carries the invocation of its update", func(t *testing.T) {
repoPath := t.TempDir()
- seq := appendPlain(t, repoPath, "refs/heads/main", landOID(9))
recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
intents := intentsOf(t, repoPath)
require.Len(t, intents, 1)
- require.Equal(t, seq, intents[0].Head)
+ require.Equal(t, invocationOf(landOID(1)), intents[0].Invocation)
+ })
+
+ t.Run("a reader that knows nothing reads the file once", func(t *testing.T) {
+ repoPath := t.TempDir()
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))
+
+ read := 0
+ known, reset, err := readIntents(intentsPath(repoPath), intentsFile{}, func(landIntent) { read++ })
+ require.NoError(t, err)
+ require.False(t, reset)
+ require.Equal(t, 2, read)
+ require.Equal(t, fileSize(t, intentsPath(repoPath)), known.offset)
})
}
-// TestLandIntentsAreDurable replaces syncFile, so it does not run in parallel.
+// TestLandIntentsAreDurable replaces syncFile and syncDir, so it does not run in parallel.
func TestLandIntentsAreDurable(t *testing.T) {
t.Run("the intents are fsync'd before they return", func(t *testing.T) {
repoPath := t.TempDir()
@@ -151,77 +192,137 @@ func TestLandIntentsAreDurable(t *testing.T) {
return f.Sync()
})
- require.ErrorContains(t, RecordLandIntents(repoPath, "refs/heads/main", []Land{{Original: landOID(1), Landed: landOID(2)}}), "injected")
+ _, err := RecordLandIntents(repoPath, "refs/heads/main", "inv", []Land{{Original: landOID(1), Landed: landOID(2)}})
+ require.ErrorContains(t, err, "injected")
+ })
+
+ t.Run("the directories are synced for a file created again", func(t *testing.T) {
+ // Review m3: the repository is removed and created again at the same path. The intents
+ // file of the new repository is another file, whose directory entries are synced.
+ repoPath := t.TempDir()
+ dirSyncs := 0
+ original := syncDir
+ syncDir = func(dir string) error {
+ dirSyncs++
+ return original(dir)
+ }
+ t.Cleanup(func() { syncDir = original })
+
+ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))
+ require.Positive(t, dirSyncs, "the write that created the file syncs the directories")
+
+ dirSyncs = 0
+ recordIntent(t, repoPath, "refs/heads/main", landOID(3), landOID(4))
+ require.Zero(t, dirSyncs, "a later write to the same file does not")
+
+ require.NoError(t, os.RemoveAll(filepath.Join(repoPath, dirName)))
+ recordIntent(t, repoPath, "refs/heads/main", landOID(5), landOID(6))
+ require.Positive(t, dirSyncs, "the write that created the file again syncs the directories")
})
}
func TestLandIntentsCompaction(t *testing.T) {
t.Parallel()
- repoPath := t.TempDir()
+ // recordAndConfirm records n intents in one update each, and the changelog entries of the
+ // updates of the first confirmed of them. The entries of the next superseded intents' commits
+ // are written by other updates, with other landed commits: they do not confirm the intents.
+ recordAndConfirm := func(t *testing.T, repoPath string, n, confirmed, superseded int) []Land {
+ lands := make([]Land, n)
+ for i := range lands {
+ lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}
+ _, err := RecordLandIntents(repoPath, "refs/heads/main", invocationOf(lands[i].Original), lands[i:i+1])
+ require.NoError(t, err)
+ }
- // Enough intents to compact the file: half of them confirmed by the changelog, a quarter
- // superseded by a later land, a quarter never committed and never landed again.
- const n = 2000
- lands := make([]Land, n)
- for i := range lands {
- lands[i] = Land{Original: landOID(10000 + i), Landed: landOID(20000 + i)}
- }
- require.NoError(t, RecordLandIntents(repoPath, "refs/heads/main", lands))
- require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
+ var records []Record
+ for i := 0; i < confirmed+superseded; i++ {
+ land, invocation := lands[i], invocationOf(lands[i].Original)
+ if i >= confirmed {
+ land, invocation = Land{Original: lands[i].Original, Landed: landOID(30000 + i)}, "inv-other"
+ }
+ records = append(records, Record{
+ Meta: Meta{Operation: "land", LandReference: "refs/heads/main", LandInvocation: invocation, Lands: []Land{land}},
+ Updates: []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)}},
+ })
+ }
+ _, err := AppendGroup(repoPath, records)
+ require.NoError(t, err)
- var confirmed []Land
- for i := 0; i < n/2; i++ {
- confirmed = append(confirmed, lands[i])
- }
- for i := n / 2; i < 3*n/4; i++ {
- confirmed = append(confirmed, Land{Original: lands[i].Original, Landed: landOID(30000 + i)})
+ return lands
}
- updates := make([]Update, 0, len(confirmed))
- for i := range confirmed {
- updates = append(updates, Update{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(40000 + i)})
- }
- // One entry per land, each carrying its own land.
- records := make([]Record, len(confirmed))
- for i, land := range confirmed {
- records[i] = Record{Meta: Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{land}}, Updates: updates[i : i+1]}
- }
- _, err := AppendGroup(repoPath, records)
- require.NoError(t, err)
- lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+ t.Run("an intent goes once the entry of its own update is durable", func(t *testing.T) {
+ repoPath := t.TempDir()
+ const n = 2000
+ lands := recordAndConfirm(t, repoPath, n, 3*n/4, n/8)
+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
- kept := intentsOf(t, repoPath)
- require.Len(t, kept, n/4, "only the intents that the changelog neither confirms nor supersedes are kept")
- for _, intent := range kept {
- require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)
- }
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
- // The kept intents, more than the size from which a file is compacted, are not rewritten by
- // every lookup.
- require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
- compacted, err := os.Stat(intentsPath(repoPath))
- require.NoError(t, err)
- lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
- again, err := os.Stat(intentsPath(repoPath))
- require.NoError(t, err)
- require.True(t, os.SameFile(compacted, again))
+ kept := intentsOf(t, repoPath)
+ require.Len(t, kept, n/4, "the intents whose own update has no entry stay, also those of a recorded commit")
+ for _, intent := range kept {
+ require.True(t, strings.Compare(intent.Original, landOID(10000+3*n/4)) >= 0)
+ }
- // Every land is still answered as before the compaction, also by a new process.
- forgetLandsIndex(t, repoPath)
- for i, land := range lands {
- records := lookupAll(t, repoPath, "refs/heads/main", land.Original)
- require.Len(t, records, 1)
- switch {
- case i < n/2:
- require.Equal(t, land.Landed, records[0].Landed)
- require.NotZero(t, records[0].Sequence)
- case i < 3*n/4:
- require.Equal(t, landOID(30000+i), records[0].Landed)
- default:
- require.Equal(t, LandRecord{Landed: land.Landed}, records[0])
+ // Every land is still answered as before the compaction, also by a new process.
+ forgetLandsIndex(t, repoPath)
+ for i, land := range lands {
+ records := lookupAll(t, repoPath, "refs/heads/main", land.Original)
+ switch {
+ case i < 3*n/4:
+ require.Len(t, records, 1)
+ require.Equal(t, land.Landed, records[0].Landed)
+ require.NotZero(t, records[0].Sequence)
+ case i < 3*n/4+n/8:
+ require.Equal(t, []LandRecord{{Sequence: records[0].Sequence, Landed: landOID(30000 + i)}, {Landed: land.Landed}}, records)
+ default:
+ require.Equal(t, []LandRecord{{Landed: land.Landed}}, records)
+ }
}
- }
+ })
+
+ t.Run("intents that must stay are not rewritten again", func(t *testing.T) {
+ // Review m4: after a compaction, also in a new process, the intents that stay are not
+ // rewritten while too few of them may go.
+ repoPath := t.TempDir()
+ const n = 2000
+ recordAndConfirm(t, repoPath, n, n/2, 0)
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+ require.Len(t, intentsOf(t, repoPath), n/2)
+ require.Greater(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
+
+ compacted, err := os.Stat(intentsPath(repoPath))
+ require.NoError(t, err)
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+ forgetLandsIndex(t, repoPath)
+ lookupAll(t, repoPath, "refs/heads/main", landOID(10000))
+ again, err := os.Stat(intentsPath(repoPath))
+ require.NoError(t, err)
+ require.True(t, os.SameFile(compacted, again))
+ })
+
+ t.Run("the intents of lands that never look up are compacted too", func(t *testing.T) {
+ // Review m4: fast-forward lands never look up an earlier land. The append asks for the
+ // compaction once the file grew, and CompactLandIntents does it.
+ repoPath := t.TempDir()
+ const n = 1000
+ var due bool
+ for i := 0; i < n; i++ {
+ land := Land{Original: landOID(10000 + i), Landed: landOID(10000 + i)}
+ compact, err := RecordLandIntents(repoPath, "refs/heads/main", invocationOf(land.Original), []Land{land})
+ require.NoError(t, err)
+ confirmLand(t, repoPath, "refs/heads/main", land.Original, land.Landed, invocationOf(land.Original))
+ if compact {
+ due = true
+ require.NoError(t, CompactLandIntents(repoPath))
+ }
+ }
+
+ require.True(t, due, "the append asked for a compaction")
+ require.Less(t, fileSize(t, intentsPath(repoPath)), int64(compactIntentsMinSize))
+ })
}
func TestLandsIndexCompaction(t *testing.T) {
diff --git a/internal/graft/changelog/log.go b/internal/graft/changelog/log.go
index 661cdb2..a1dee8b 100644
--- a/internal/graft/changelog/log.go
+++ b/internal/graft/changelog/log.go
@@ -97,6 +97,11 @@ type Entry struct {
// recorded on every land path and are never cut (see Land, WithLands and doc/graft/PLAN11.md).
// They are empty for updates that are not lands.
Lands []Land `json:"lands,omitempty"`
+ // LandInvocation identifies the update that landed Lands: the invocation of its reference
+ // transaction (see gitcmd.ChangelogMeta.Invocation), which its land intents carry too. A land
+ // intent is confirmed, and may go, once the entry of its own update is durable. It is empty
+ // without Lands.
+ LandInvocation string `json:"land_invocation,omitempty"`
}
// entryFields is Entry without its JSON methods.
@@ -175,6 +180,9 @@ type Meta struct {
// LandReference is the branch that Lands landed on. Only the entry of the update of that
// branch carries them (see LandsOf).
LandReference string
+ // LandInvocation identifies the update that recorded Lands, as its land intents do (see
+ // Entry.LandInvocation).
+ LandInvocation string
// Lands are the identities of the changes that the updates landed, see Entry.Lands.
Lands []Land
}
@@ -482,6 +490,7 @@ func (s *logState) appendLocked(repoPath, path string, records []Record, entries
Members: normalizeMembers(meta.Members),
MembersTruncated: meta.MembersTruncated,
Lands: LandsOf(meta, update.Reference),
+ LandInvocation: LandInvocationOf(meta, update.Reference),
}
line, err := json.Marshal(entry)
diff --git a/internal/graft/corestore/sqlstore/changeset.go b/internal/graft/corestore/sqlstore/changeset.go
index c83cdd9..a1658a6 100644
--- a/internal/graft/corestore/sqlstore/changeset.go
+++ b/internal/graft/corestore/sqlstore/changeset.go
@@ -256,6 +256,7 @@ func buildChangelogEntries(repo *repoState, meta *changelog.Meta, now time.Time,
Members: slices.Clone(meta.Members),
MembersTruncated: meta.MembersTruncated,
Lands: changelog.LandsOf(*meta, u.Name),
+ LandInvocation: changelog.LandInvocationOf(*meta, u.Name),
})
}