Skip to content

Git designed for AI agents.

changelog: a land lookup reads only the entries appended since the last one

Federico Eliaeb4251d0069d
diff --git a/internal/graft/changelog/lands_bench_test.go b/internal/graft/changelog/lands_bench_test.go
new file mode 100644
index 0000000..b65085e
--- /dev/null
+++ b/internal/graft/changelog/lands_bench_test.go
@@ -0,0 +1,44 @@
+package changelog
+
+import (
+ "fmt"
+ "testing"
+)
+
+// BenchmarkLookupLandAfterOneAppend measures a lookup that catches up one new entry of a log of the
+// given size, as a land that diverges does after the land before it.
+func BenchmarkLookupLandAfterOneAppend(b *testing.B) {
+ for _, size := range []int{10, 250, 2000} {
+ b.Run(fmt.Sprintf("log of %d", size), func(b *testing.B) {
+ repoPath := b.TempDir()
+ records := make([]Record, size)
+ for i := range records {
+ records[i] = Record{
+ Meta: Meta{Operation: "land", LandReference: "refs/heads/main", LandInvocation: fmt.Sprint("inv-", i), Lands: []Land{{Original: landOID(1000000 + i), Landed: landOID(2000000 + i)}}},
+ Updates: []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: landOID(2000000 + i)}},
+ }
+ }
+ if _, err := AppendGroup(repoPath, records); err != nil {
+ b.Fatal(err)
+ }
+ if _, err := LookupLand(repoPath, "refs/heads/main", landOID(1)); err != nil {
+ b.Fatal(err)
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ b.StopTimer()
+ land := Land{Original: landOID(3000000 + i), Landed: landOID(4000000 + i)}
+ if _, err := Append(repoPath, Meta{Operation: "land", LandReference: "refs/heads/main", Lands: []Land{land}},
+ []Update{{Reference: "refs/heads/main", OldOID: landOID(0), NewOID: land.Landed}}); err != nil {
+ b.Fatal(err)
+ }
+ b.StartTimer()
+
+ if _, err := LookupLand(repoPath, "refs/heads/main", land.Original); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/graft/changelog/lands_index.go b/internal/graft/changelog/lands_index.go
index 45bcd4b..5b7286b 100644
--- a/internal/graft/changelog/lands_index.go
+++ b/internal/graft/changelog/lands_index.go
@@ -100,6 +100,9 @@ type landsIndex struct {
mark landsMark
// logDevice and logInode identify the log file that records was read from.
logDevice, logInode uint64
+ // markOffset is the offset in the log of the line of the mark's entry, where the next
+ // catch-up starts; -1 when it is not known.
+ markOffset int64
// records maps the lands of the log entries up to mark to their last record.
records map[landsIndexKey]indexRecord
// confirmed holds the invocations of the land updates whose changelog entries the index read
@@ -360,6 +363,7 @@ func (i *landsIndex) catchUpLog(repoPath string) error {
i.fileSize += int64(len(encoded))
}
+ i.markOffset = block.markOffset
i.logDevice, i.logInode = head.device, head.inode
i.loaded = true
@@ -499,13 +503,15 @@ type landsBlockRecord struct {
type landsBlock struct {
records []landsBlockRecord
mark landsMark
+ // markOffset is the offset in the log of the line of the mark's entry, -1 when unknown.
+ markOffset int64
}
// scanLog reads the log entries after the mark of the index and returns their lands as a block. It
// first checks that the log entry at the mark is the one the mark names, and returns
// errLandsIndexStale when it is not. The caller must hold i.mu.
func (i *landsIndex) scanLog(repoPath string) (landsBlock, error) {
- block := landsBlock{mark: i.mark}
+ block := landsBlock{mark: i.mark, markOffset: i.markOffset}
after := i.mark.sequence
verified := after == 0
@@ -513,12 +519,15 @@ func (i *landsIndex) scanLog(repoPath string) (landsBlock, error) {
after--
}
- if err := Scan(repoPath, after, func(entry Entry) (bool, error) {
+ // The scan starts at the line of the mark's entry when the index knows it, so that a lookup
+ // reads only the entries appended since the last one.
+ if err := scanFrom(repoPath, after, i.markOffset, func(entry Entry, offset int64) (bool, error) {
if !verified {
if !i.mark.matches(entry) {
return false, errLandsIndexStale
}
verified = true
+ block.markOffset = offset
return true, nil
}
@@ -532,6 +541,7 @@ func (i *landsIndex) scanLog(repoPath string) (landsBlock, error) {
})
}
block.mark = landsMark{sequence: entry.Sequence, reference: entry.Reference, newOID: entry.NewOID}
+ block.markOffset = offset
return true, nil
}); err != nil {
@@ -598,6 +608,7 @@ func (b landsBlock) encode() ([]byte, error) {
// reset empties the in-memory index of a file of the given size.
func (i *landsIndex) reset(fileSize int64) {
i.mark = landsMark{}
+ i.markOffset = -1
i.records = map[landsIndexKey]indexRecord{}
i.confirmed = map[string]struct{}{}
i.recordLines = 0
diff --git a/internal/graft/changelog/log.go b/internal/graft/changelog/log.go
index a1dee8b..0433778 100644
--- a/internal/graft/changelog/log.go
+++ b/internal/graft/changelog/log.go
@@ -907,6 +907,16 @@ func lockFile(path string) (func(), error) {
// The scan starts at the closest index point before after, so its cost depends
// on the entries that follow after and not on the size of the log.
func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {
+ return scanFrom(repoPath, after, -1, func(entry Entry, _ int64) (bool, error) {
+ return fn(entry)
+ })
+}
+
+// scanFrom is Scan that also gives fn the offset of the line of each entry, and that starts at the
+// offset hint when it is not negative and its line holds the entry with the sequence after+1: a
+// caller that knows that offset from an earlier scan reads only the entries it asks for. A hint
+// whose line does not hold that entry (the log was replaced) is ignored.
+func scanFrom(repoPath string, after uint64, hint int64, fn func(Entry, int64) (bool, error)) error {
path := Path(repoPath)
state, release := acquireState(path)
@@ -927,6 +937,9 @@ func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {
}
offset := state.scanOffset(f, after)
+ if hint >= 0 && hint < end && lineHolds(f, hint, after+1) {
+ offset = hint
+ }
if offset >= end {
return nil
}
@@ -955,7 +968,7 @@ func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {
continue
}
- more, err := fn(entry)
+ more, err := fn(entry, lineOffset)
if err != nil {
return err
}
@@ -965,6 +978,17 @@ func Scan(repoPath string, after uint64, fn func(Entry) (bool, error)) error {
}
}
+// lineHolds reports whether the line of f at offset holds the entry with the given sequence.
+func lineHolds(f *os.File, offset int64, sequence uint64) bool {
+ line, err := bufio.NewReader(io.NewSectionReader(f, offset, 1<<62)).ReadBytes('\n')
+ if err != nil {
+ return false
+ }
+
+ var entry Entry
+ return json.Unmarshal(line, &entry) == nil && entry.Sequence == sequence
+}
+
// LastSequence returns the sequence of the last durable entry of the log of the
// repository whose git directory is repoPath, 0 when the log is empty or does
// not exist. An entry whose append has not fsync'd it yet is not included (see