Skip to content

Git designed for AI agents.

Split the Preparing phase of authoritative writes into its steps

Federico Elia1d37569e57db
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 76dfea9..50b8adc 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -710,6 +710,7 @@ without a single divergence.
|---|---|
| `graft_storage_changesets_total{mode,result}` | Reference transactions handled by the write path; results `committed`, `noop`, `conflict`, `error`, `skipped`. |
| `graft_storage_preparing_seconds` | Time of the write path in the Preparing phase, before git locks the references (pack build and upload). |
+| `graft_storage_preparing_step_seconds{step}` | The steps of the Preparing phase in authoritative mode: `record` (the read of the repository record), `walk` (the objects reachable from the new tips and not from a local reference, until their list ends; observed for every transaction with new tips), `pack` (writing and hashing a non-empty pack) and `upload` (to the bucket). |
| `graft_storage_prepared_seconds` | Time of the write path in the Prepared phase, while git holds the reference locks (expected values and database commit; also the pack when the Preparing phase did not prepare it). |
| `graft_storage_pack_bytes_total` | Bytes of packs uploaded. |
| `graft_storage_shadow_divergences_total` | Shadow mode changesets recorded despite a mismatch. |
diff --git a/internal/graft/corestore/gitops/gitops.go b/internal/graft/corestore/gitops/gitops.go
index b20386e..a6d4f36 100644
--- a/internal/graft/corestore/gitops/gitops.go
+++ b/internal/graft/corestore/gitops/gitops.go
@@ -24,6 +24,8 @@ import (
"path/filepath"
"sort"
"strings"
+ "sync/atomic"
+ "time"
"github.com/feder-cr/git-ai/internal/command"
"github.com/feder-cr/git-ai/internal/git"
@@ -51,6 +53,10 @@ type Pack struct {
Size int64
// Objects is the number of objects in the pack.
Objects int
+ // Walk is the time from the start of the walk until its object list ended:
+ // until pack-objects read the end of the list, or, for an empty list, until
+ // rev-list exited. The rest of the build is the packing of the objects.
+ Walk time.Duration
}
// Empty reports whether the pack has no objects (and no file).
@@ -239,6 +245,8 @@ func buildPack(ctx context.Context, repo *localrepo.Repo, revList gitcmd.Command
ctx, cancel := context.WithCancel(ctx)
defer cancel()
+ start := time.Now()
+
var revStderr bytes.Buffer
revOpts := []gitcmd.CmdOpt{gitcmd.WithSetupStdout(), gitcmd.WithStderr(&revStderr)}
if stdin != nil {
@@ -261,8 +269,9 @@ func buildPack(ctx context.Context, repo *localrepo.Repo, revList gitcmd.Command
if err := revCmd.Wait(); err != nil {
return Pack{}, structerr.New("rev-list: %w", err).WithMetadata("stderr", revStderr.String())
}
- return Pack{}, nil
+ return Pack{Walk: time.Since(start)}, nil
}
+ listEnd := &endReader{r: objectList}
file, err := os.CreateTemp(tmpDir, "graft-pack-*")
if err != nil {
@@ -288,7 +297,7 @@ func buildPack(ctx context.Context, repo *localrepo.Repo, revList gitcmd.Command
gitcmd.Flag{Name: "--stdout"},
},
},
- gitcmd.WithStdin(objectList),
+ gitcmd.WithStdin(listEnd),
gitcmd.WithStdout(io.MultiWriter(file, hasher)),
gitcmd.WithStderr(&packStderr),
)
@@ -337,9 +346,38 @@ func buildPack(ctx context.Context, repo *localrepo.Repo, revList gitcmd.Command
SHA256: sum,
Size: info.Size(),
Objects: int(binary.BigEndian.Uint32(header[8:12])),
+ Walk: listEnd.endedAfter(start),
}, nil
}
+// endReader is a reader that records when the reader it wraps ended. It is read
+// by the goroutine that feeds the standard input of a command, and asked once the
+// command has exited.
+type endReader struct {
+ r io.Reader
+ // end is the time the reader ended, in Unix nanoseconds; 0 until then.
+ end atomic.Int64
+}
+
+func (e *endReader) Read(p []byte) (int, error) {
+ n, err := e.r.Read(p)
+ if errors.Is(err, io.EOF) {
+ e.end.CompareAndSwap(0, time.Now().UnixNano())
+ }
+ return n, err
+}
+
+// endedAfter returns the time from start until the reader ended. A command that
+// exited without reading its input to the end (pack-objects reads all of it
+// before it writes) has no end, and then the whole time since start counts.
+func (e *endReader) endedAfter(start time.Time) time.Duration {
+ end := e.end.Load()
+ if end == 0 {
+ return time.Since(start)
+ }
+ return time.Unix(0, end).Sub(start)
+}
+
// IndexPack adds the pack read from r to the repository with
// `git index-pack --stdin` (which verifies every object), so that the objects
// are in objects/pack of the repository afterwards. An empty reader is an
diff --git a/internal/graft/corestore/gitops/gitops_test.go b/internal/graft/corestore/gitops/gitops_test.go
index b2ad2ce..c888712 100644
--- a/internal/graft/corestore/gitops/gitops_test.go
+++ b/internal/graft/corestore/gitops/gitops_test.go
@@ -131,6 +131,7 @@ func TestBuildPackOnlyNewObjects(t *testing.T) {
require.NoError(t, err)
require.False(t, pack.Empty())
require.Equal(t, 3, pack.Objects, "commit, tree and blob of c2 only")
+ require.Positive(t, pack.Walk, "the walk of the objects is timed")
require.Equal(t, 1, countEntries(t, tmpDir))
require.True(t, strings.HasPrefix(filepath.Base(pack.Path), "graft-pack-"))
readPack(t, pack)
@@ -201,10 +202,12 @@ func TestBuildPackEmpty(t *testing.T) {
require.NoError(t, err)
require.True(t, pack.Empty())
require.Equal(t, 0, pack.Objects)
+ require.Positive(t, pack.Walk, "an empty walk is timed too")
pack, err = gitops.BuildPack(f.ctx, f.repo, nil, tmpDir)
require.NoError(t, err)
require.True(t, pack.Empty())
+ require.Zero(t, pack.Walk, "without a tip there is no walk")
pack, err = gitops.BuildPack(f.ctx, f.repo, []git.ObjectID{f.hash.ZeroOID}, tmpDir)
require.NoError(t, err)
diff --git a/internal/graft/corestore/voter/preparing_test.go b/internal/graft/corestore/voter/preparing_test.go
index 8a24ca3..8f0819d 100644
--- a/internal/graft/corestore/voter/preparing_test.go
+++ b/internal/graft/corestore/voter/preparing_test.go
@@ -78,6 +78,15 @@ func TestPreparingUploadsThePackOnce(t *testing.T) {
require.Equal(t, corestore.Target(f.c.String()), f.dbRef(t, ref))
require.Equal(t, float64(changeset.PackBytes), metricValue(t, f.voter, "graft_storage_pack_bytes_total", nil))
require.Equal(t, float64(1), metricValue(t, f.voter, "graft_storage_preparing_seconds", nil))
+ // Authoritative mode observes every step of the Preparing phase once; the
+ // shadow mode, which builds its pack another way, none.
+ steps := float64(0)
+ if mode == corestore.ModeAuthoritative {
+ steps = 1
+ }
+ for _, step := range []string{"record", "walk", "pack", "upload"} {
+ require.Equal(t, steps, metricValue(t, f.voter, "graft_storage_preparing_step_seconds", map[string]string{"step": step}), step)
+ }
f.voter.Committed(f.ctx, f.payload, input)
require.Equal(t, []noteCall{{repoPath: f.repoPath, repoID: f.repoID, seq: f.seq(2)}}, f.applier.calls)
@@ -98,6 +107,8 @@ func TestPreparedWithoutPreparing(t *testing.T) {
require.NoError(t, f.prepared(line(f.zero, f.b.String(), ref)))
require.Equal(t, 1, blobs.count(), "Prepared uploads the pack itself")
+ require.Zero(t, metricValue(t, f.voter, "graft_storage_preparing_step_seconds", nil),
+ "the steps are those of the Preparing phase, not of a pack that Prepared builds")
require.Equal(t, blobs.puts[0], f.lastChangeset(t).PackKey)
require.Equal(t, corestore.Target(f.b.String()), f.dbRef(t, ref))
})
diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.go
index 8f3342c..b700f84 100644
--- a/internal/graft/corestore/voter/voter.go
+++ b/internal/graft/corestore/voter/voter.go
@@ -255,11 +255,12 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, ch
if err != nil {
return nil, err
}
+ v.observeStep(stepRecord, time.Since(readAt))
if record.HeadSeq == 0 {
return nil, nil
}
- pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)
+ pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates, true)
if err != nil {
return nil, err
}
@@ -457,7 +458,7 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
if prepared != nil && prepared.ok && prepared.repoID == record.ID && prepared.usable(time.Now()) {
pack = prepared.pack
} else {
- pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)
+ pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates, false)
if err != nil {
return v.unavailable(err)
}
@@ -679,7 +680,10 @@ func newTips(updates []inputUpdate) []git.ObjectID {
// local reference reaches (invariant I1, gitops.BuildPack); in shadow mode,
// everything that the values the database records reach (buildPackAgainst),
// because local references can hold values the storage core never recorded.
-func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, record corestore.Repository, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {
+//
+// preparing tells that the Preparing phase builds the pack: its steps are then
+// observed (observeStep).
+func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, record corestore.Repository, readAt time.Time, updates []inputUpdate, preparing bool) (corestore.PackUpload, error) {
repoID := record.ID
tips := newTips(updates)
if len(tips) == 0 {
@@ -698,7 +702,14 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag
var pack gitops.Pack
if v.core.Authoritative() {
+ built := time.Now()
pack, err = gitops.BuildPack(ctx, repo, tips, tmpDir)
+ if err == nil && preparing {
+ v.observeStep(stepWalk, pack.Walk)
+ if !pack.Empty() {
+ v.observeStep(stepPack, time.Since(built)-pack.Walk)
+ }
+ }
} else {
var known map[string]corestore.Target
known, _, err = v.core.Meta.Refs(ctx, repoID)
@@ -722,15 +733,45 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag
}
defer func() { _ = r.Close() }()
+ uploading := time.Now()
upload, err := v.core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, r)
if err != nil {
return corestore.PackUpload{}, err
}
+ if preparing {
+ v.observeStep(stepUpload, time.Since(uploading))
+ }
v.metrics.packBytes.Add(float64(pack.Size))
return upload, nil
}
+// The steps of the Preparing phase that graft_storage_preparing_step_seconds
+// observes (doc/graft/STORAGE.md).
+const (
+ // stepRecord is the read of the repository record.
+ stepRecord = "record"
+ // stepWalk is the walk of the objects that the pack holds, until its object
+ // list ended (gitops.Pack.Walk). It is observed for every transaction with new
+ // tips, an empty walk included.
+ stepWalk = "walk"
+ // stepPack is the packing of the walked objects into the temporary file,
+ // hashed while it is written. It is observed for non-empty packs only.
+ stepPack = "pack"
+ // stepUpload is the upload of the pack to the bucket.
+ stepUpload = "upload"
+)
+
+// observeStep observes d in step of graft_storage_preparing_step_seconds. Only
+// authoritative mode observes the steps: the shadow mode builds its packs
+// another way (buildPackAgainst).
+func (v *Voter) observeStep(step string, d time.Duration) {
+ if !v.core.Authoritative() {
+ return
+ }
+ v.metrics.preparingSteps.WithLabelValues(step).Observe(d.Seconds())
+}
+
// fault calls FaultHook and exits the process when CrashEnv selects the point.
func (v *Voter) fault(point string) error {
if FaultHook != nil {
@@ -1191,7 +1232,10 @@ type metrics struct {
localFailures prometheus.Counter
packBytes prometheus.Counter
preparing prometheus.Histogram
- prepared prometheus.Histogram
+ // preparingSteps splits the Preparing phase of authoritative writes into its
+ // steps (observeStep).
+ preparingSteps *prometheus.HistogramVec
+ prepared prometheus.Histogram
}
func newMetrics() *metrics {
@@ -1229,6 +1273,13 @@ func newMetrics() *metrics {
Help: "Time spent in the Preparing phase of the storage core write path (pack build and upload, before git takes the reference locks).",
Buckets: prometheus.DefBuckets,
}),
+ preparingSteps: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "graft_storage_preparing_step_seconds",
+ Help: "Time of each step of the Preparing phase of authoritative writes: record (the read of the " +
+ "repository record), walk (the objects of the pack), pack (writing and hashing a non-empty pack) " +
+ "and upload (to the bucket).",
+ Buckets: []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5},
+ }, []string{"step"}),
prepared: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "graft_storage_prepared_seconds",
Help: "Time spent in the Prepared phase of the storage core write path, while git holds the reference locks.",
@@ -1247,6 +1298,7 @@ func (m *metrics) collectors() []prometheus.Collector {
m.localFailures,
m.packBytes,
m.preparing,
+ m.preparingSteps,
m.prepared,
}
}