Git designed for AI agents.
package e2e
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"github.com/feder-cr/git-ai/internal/featureflag"
"github.com/feder-cr/git-ai/internal/git/gittest"
"github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/service"
"github.com/feder-cr/git-ai/internal/gitaly/service/setup"
"github.com/feder-cr/git-ai/internal/gitlab"
"github.com/feder-cr/git-ai/internal/graft/landlane"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
"github.com/feder-cr/git-ai/internal/testhelper/testserver"
"github.com/feder-cr/git-ai/sdk"
)
// The phase-4 end-to-end tests (WP39) cover the batch latency work: the graph
// ancestry checks must not change the outcome of a batch, the adaptive batch
// sizing and early completion of the executor, and the plain land of a batch
// of one. The tests run against the real executor of WP35 and the lanes of
// WP37.
// phase4GaugeName is the gauge of the effective maximum batch size, see
// internal/graft/landlane.
const phase4GaugeName = "graft_land_lane_effective_max_size"
// phase4NextEffectiveMax is the model of the adaptive batch sizing of WP37
// that the landlane gauge graft_land_lane_effective_max_size must follow.
// After a batch that took longer than the target the effective size halves,
// never below one. After a batch that took less than half the target it
// grows by a quarter, at least one, never above the configured maximum.
// Otherwise it stays. A target of zero turns the adaptation off.
func phase4NextEffectiveMax(current, maxSize int, duration, target time.Duration) int {
switch {
case target <= 0:
return current
case duration > target:
return max(1, current/2)
case duration < target/2:
return min(maxSize, current+max(1, current/4))
default:
return current
}
}
// TestPhase4AdaptiveSizingModel checks the model of the adaptive sizing that
// TestPhase4AdaptiveSizingVisible compares the gauge with. It needs no server.
func TestPhase4AdaptiveSizingModel(t *testing.T) {
t.Parallel()
const (
maxSize = 8
target = 500 * time.Millisecond
slow = 2 * target
fast = target / 4
steady = target * 3 / 4
)
steps := []struct {
desc string
duration time.Duration
want int
}{
{"slow halves", slow, 4},
{"slow halves again", slow, 2},
{"slow to the floor", slow, 1},
{"slow stays at one", slow, 1},
{"fast grows by one", fast, 2},
{"fast grows by one again", fast, 3},
{"fast grows by one more", fast, 4},
{"steady keeps the size", steady, 4},
{"fast grows to five", fast, 5},
{"fast grows to six", fast, 6},
{"fast grows to seven", fast, 7},
{"fast grows to the maximum", fast, 8},
{"fast stays at the maximum", fast, 8},
}
current := maxSize
for _, step := range steps {
current = phase4NextEffectiveMax(current, maxSize, step.duration, target)
require.Equal(t, step.want, current, step.desc)
}
require.Equal(t, 5, phase4NextEffectiveMax(5, maxSize, slow, 0), "a zero target turns the adaptation off")
}
// phase4MixedRun lands private disjoint changes and shared changes, all made
// from one base and all touching the shared log, on main at the same time,
// with workerGraft tokens. It returns the tally of the lands and the files
// that the agents added to main.
func phase4MixedRun(t *testing.T, graft config.Graft, tokens map[string]string, private, shared int) (loadTally, []string) {
t.Helper()
s := startGitaly(t, graft, sharedTestToken)
ctx := s.ctx
base := writeMain(t, s, map[string]string{sharedLogPath: sharedLogHeader})
agents := private + shared
repos := make([]*sdk.Repo, agents)
commits := make([]string, agents)
for i := range repos {
name := workerName(i)
repos[i] = s.repoAs(t, tokens[name])
var err error
commits[i], err = sharedChange(ctx, repos[i], name, base, 1, i >= private)
require.NoError(t, err)
}
tally := runLoad(agents, func(agent int) error {
_, err := landOn(ctx, repos[agent], "main", commits[agent], false)
return err
})
require.NoError(t, tally.check(agents))
files := strings.Split(s.git(t, "ls-tree", "-r", "--name-only", "main", "agents"), "\n")
return tally, files
}
// TestPhase4GraphAncestryMatchesGit lands the same mixed batch with the graph
// ancestry checks on and off. The tallies must be equal and equal to the
// expected outcome, and the same private changes must be on main.
func TestPhase4GraphAncestryMatchesGit(t *testing.T) {
t.Parallel()
const private, shared = 17, 3
var tallies [2]loadTally
var files [2][]string
for i, on := range []bool{false, true} {
graft, tokens := workerGraft(private + shared)
graft.Land.Batch.Window = "300ms"
graft.Land.GraphAncestry = on
tallies[i], files[i] = phase4MixedRun(t, graft, tokens, private, shared)
}
want := expectedMixedOutcome(private, shared)
require.Equal(t, want, tallies[0], "the outcome without graph ancestry")
require.Equal(t, want, tallies[1], "the outcome with graph ancestry")
for run, list := range files {
var privateOnMain, sharedOnMain int
for i := 0; i < private+shared; i++ {
if !slices.Contains(list, fmt.Sprintf("agents/%s/1.txt", workerName(i))) {
continue
}
if i < private {
privateOnMain++
} else {
sharedOnMain++
}
}
require.Equal(t, private, privateOnMain, "run %d: every private change is on main", run)
require.Equal(t, 1, sharedOnMain, "run %d: exactly one shared change is on main", run)
}
}
// TestPhase4BatchOfOneIsLand checks that a change that lands alone in a batch
// is recorded as an ordinary land by the real actor. The land keeps its single
// member, so the original commit maps to the commit it landed as, and the
// policy check of the land is the one of an unbatched land.
func TestPhase4BatchOfOneIsLand(t *testing.T) {
t.Parallel()
s := startExample(t)
repo := s.repoAs(t, exampleTokens["planner"])
ctx := s.ctx
base := writeMain(t, s, map[string]string{"README.md": "single\n"})
const branch = "phase4-single"
_, err := landAs(ctx, repo, "planner", branch, base)
require.NoError(t, err)
commit, err := commitFile(ctx, repo, "planner", "topic/phase4-single", base, "phase4-single.txt", "one\n")
require.NoError(t, err)
res, err := landOn(ctx, repo, branch, commit, false)
require.NoError(t, err)
require.Equal(t, int32(1), res.Batch.Size, "the change is a batch of one")
var landed []*sdk.Changeset
for _, cs := range allChangesets(ctx, t, repo) {
if string(cs.GetReference()) == "refs/heads/"+branch && len(cs.GetMembers()) == 1 {
landed = append(landed, cs)
}
}
require.Len(t, landed, 1, "exactly one changeset records the batch of one")
cs := landed[0]
require.Equal(t, "land", cs.GetOperation(), "a batch of one is a land")
require.Equal(t, "planner", cs.GetActor(), "the actor is the real land actor")
member := cs.GetMembers()[0]
require.Equal(t, commit, member.GetOriginalCommitId(), "the member is the submitted commit")
require.Equal(t, res.LandedCommitID, member.GetCommitId(), "the member is the landed commit")
require.Equal(t, "planner", member.GetActor(), "the member is attributed to its agent")
}
// startGitalyWithLanes starts a standalone Gitaly like startGitaly, except that
// its land batches run in lanes that the test owns, so that the test can read
// the metrics of the lanes. The lanes are closed when the test ends.
func startGitalyWithLanes(t *testing.T, graft config.Graft, lanes *landlane.Shared) gitalyStack {
t.Helper()
ctx := featureflag.OutgoingCtxWithFeatureFlag(testhelper.Context(t), featureflag.GPGSigning, false)
cfg := testcfg.Build(t)
cfg.Auth.Token = sharedTestToken
cfg.Graft = graft
cfg.Graft.Standalone = true
testcfg.BuildGitalySSH(t, cfg)
testcfg.BuildGitalyHooks(t, cfg)
registrar := func(srv *grpc.Server, deps *service.Dependencies) {
deps.GraftLandLanes = lanes
setup.RegisterAll(srv, deps)
}
addr := testserver.RunGitalyServer(t, cfg, registrar,
testserver.WithGitLabClient(gitlab.NewStandaloneClient(cfg.Graft, testhelper.SharedLogger(t))),
)
cfg.SocketPath = addr
// Only the setup call carries gitaly-servers metadata, see startGitaly.
setupCtx := testhelper.MergeOutgoingMetadata(ctx, testcfg.GitalyServersMetadataFromCfg(t, cfg))
repoProto, repoPath := gittest.CreateRepository(t, setupCtx, cfg)
return gitalyStack{
ctx: ctx,
cfg: cfg,
addr: addr,
storage: cfg.Storages[0].Name,
relPath: repoProto.GetRelativePath(),
repoPath: repoPath,
}
}
// newTestLanes builds the lanes of a configuration, as the server does, and
// closes them when the test ends.
func newTestLanes(t *testing.T, graft config.Graft) *landlane.Shared {
t.Helper()
return newTestLanesWithClock(t, graft, nil)
}
// newTestLanesWithClock is newTestLanes with the clock that measures the batches
// (landlane.Config.Clock); nil is the time of the system.
func newTestLanesWithClock(t *testing.T, graft config.Graft, clock func() time.Time) *landlane.Shared {
t.Helper()
cfg, err := landlane.ConfigFromGraft(graft.Land.Batch)
require.NoError(t, err)
cfg.Clock = clock
lanes := landlane.NewShared(cfg)
t.Cleanup(lanes.Close)
return lanes
}
// effectiveMaxSize reads the gauge of the effective maximum batch size of the
// lanes, summed over their lanes.
func effectiveMaxSize(t *testing.T, lanes *landlane.Shared) float64 {
t.Helper()
registry := prometheus.NewRegistry()
require.NoError(t, registry.Register(lanes))
families, err := registry.Gather()
require.NoError(t, err)
for _, family := range families {
if family.GetName() == phase4GaugeName {
require.Len(t, family.GetMetric(), 1)
return family.GetMetric()[0].GetGauge().GetValue()
}
}
require.FailNow(t, "the gauge is not exported", "missing %s", phase4GaugeName)
return 0
}
// stepClock is a clock for the lanes (landlane.Config.Clock) that moves on by
// step at every reading, and stands still while step is zero. A batch reads it
// when it starts and when it ends, so the lanes measure every batch as lasting
// at least step, or nothing at all with step zero, however long it really takes.
type stepClock struct {
mu sync.Mutex
now time.Time
step time.Duration
}
func (c *stepClock) read() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
c.now = c.now.Add(c.step)
return c.now
}
func (c *stepClock) setStep(step time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.step = step
}
// TestPhase4AdaptiveSizingVisible checks that the effective maximum size gauge
// shrinks when batch updates are slower than the target latency, and grows
// back when they are fast. The bounds hold however the rounds split into
// batches: every slow batch halves the size and every fast batch grows it. The
// lanes measure the batches with a step clock, so the slow batches are slow and
// the fast ones fast whatever the load of the machine.
func TestPhase4AdaptiveSizingVisible(t *testing.T) {
t.Parallel()
const (
agents = 4
maxSize = 8
target = 2 * time.Second
// slowStep makes every batch slower than the target. The lanes read the clock a
// few times per request, so over the slow rounds the clock moves on by a few
// minutes at most, within landlane.KeyStateTTL: the lane keeps its size between
// the rounds.
slowStep = 3 * time.Second
)
graft, tokens := workerGraft(agents)
graft.Land.Batch.Window = "100ms"
graft.Land.Batch.MaxSize = maxSize
graft.Land.Batch.TargetLatency = target.String()
clock := &stepClock{now: time.Now()}
lanes := newTestLanesWithClock(t, graft, clock.read)
s := startGitalyWithLanes(t, graft, lanes)
ctx := s.ctx
base := writeMain(t, s, map[string]string{sharedLogPath: sharedLogHeader})
repos := make([]*sdk.Repo, agents)
for i := range repos {
repos[i] = s.repoAs(t, tokens[workerName(i)])
}
const branch = "adaptive"
_, err := landAs(ctx, repos[0], workerName(0), branch, base)
require.NoError(t, err)
round := 0
landRound := func() {
t.Helper()
round++
commits := make([]string, agents)
for i := range commits {
var err error
commits[i], err = commitFile(ctx, repos[i], workerName(i), fmt.Sprintf("topic/adaptive-%d-%d", round, i), base,
fmt.Sprintf("adaptive/%d/%02d.txt", round, i), "content\n")
require.NoError(t, err)
}
tally := runLoad(agents, func(agent int) error {
_, err := landOn(ctx, repos[agent], branch, commits[agent], false)
return err
})
require.Equal(t, loadTally{Landed: agents}, tally, "every change of round %d lands", round)
}
// Two rounds of slow batches. Each round has at least one batch, and every
// batch is slower than the target, so the size halves at least twice.
clock.setStep(slowStep)
landRound()
landRound()
slow := effectiveMaxSize(t, lanes)
require.LessOrEqual(t, slow, float64(maxSize/4), "two rounds of slow batches halve the size at least twice")
require.GreaterOrEqual(t, slow, float64(1), "the size never drops below one")
// Three rounds of fast batches. Each batch grows the size by at least one.
clock.setStep(0)
landRound()
landRound()
landRound()
grown := effectiveMaxSize(t, lanes)
require.Greater(t, grown, slow, "fast batches grow the effective size")
require.LessOrEqual(t, grown, float64(maxSize), "the size never exceeds the maximum")
}
// gatedUpdateHookScript is a pre-receive hook that holds the update until the
// file release exists. It gives up after five minutes, so that a test that
// never releases it fails instead of hanging; that bound only detects a
// deadlock, nothing waits for it.
func gatedUpdateHookScript(release string) string {
return fmt.Sprintf(`#!/bin/sh
while read old new ref; do :; done
i=0
while [ ! -f '%s' ]; do
i=$((i + 1))
[ "$i" -lt 6000 ] || exit 1
sleep 0.05
done
exit 0
`, release)
}
// TestPhase4EarlyCompletionPolicyDenied checks that a policy-denied request
// which shares a batch with landing members returns before the update of that
// batch, while the members wait for the update. The denial is final before
// the update, so it does not wait for it.
//
// The order is forced, not timed: the batch takes exactly the workers and the
// outsider (MaxSize is their number, the window is long, the size does not
// adapt), so they are in one batch whatever the scheduling; and the batch's
// update is held by its pre-receive hook until the test has seen the denial
// with no worker done.
func TestPhase4EarlyCompletionPolicyDenied(t *testing.T) {
t.Parallel()
const workers = 10
graft, tokens := protectedGraft(workers)
graft.Land.Batch.Window = "10m"
graft.Land.Batch.MaxSize = workers + 1
graft.Land.Batch.TargetLatency = "0s"
s := startGitaly(t, graft, sharedTestToken)
ctx := s.ctx
base := writeMain(t, s, map[string]string{sharedLogPath: sharedLogHeader})
// The workers and the outsider, which the policy denies on main, all land:
// the batch starts once all of them are queued.
names := make([]string, 0, workers+1)
for i := 0; i < workers; i++ {
names = append(names, workerName(i))
}
names = append(names, outsiderName)
repos := make([]*sdk.Repo, len(names))
commits := make([]string, len(names))
for i, name := range names {
repos[i] = s.repoAs(t, tokens[name])
var err error
commits[i], err = commitFile(ctx, repos[i], name, "topic/early-"+name, base,
fmt.Sprintf("early/%s.txt", name), "content\n")
require.NoError(t, err)
}
// The hook holds every reference update from now on, the setup commits
// above being done.
release := filepath.Join(t.TempDir(), "release")
gittest.WriteCustomHook(t, s.repoPath, "pre-receive", []byte(gatedUpdateHookScript(release)))
type outcome struct {
agent int
res *sdk.LandResult
err error
}
deniedDone := make(chan outcome, 1)
workersDone := make(chan outcome, workers)
for agent := range names {
go func() {
res, err := landOn(ctx, repos[agent], "main", commits[agent], false)
if agent == workers {
deniedDone <- outcome{agent: agent, res: res, err: err}
return
}
workersDone <- outcome{agent: agent, res: res, err: err}
}()
}
// The denial comes back while the batch's update is held: no worker is
// done.
denied := <-deniedDone
require.Equal(t, codes.PermissionDenied, sdk.Code(denied.err), "the outsider is denied: %v", denied.err)
select {
case w := <-workersDone:
t.Fatalf("worker %d returned before the update of its batch: %v", w.agent, w.err)
default:
}
require.NoError(t, os.WriteFile(release, nil, 0o600))
for range workers {
w := <-workersDone
require.NoError(t, w.err, "worker %d lands", w.agent)
// The batch size counts every request of the batch, the denied one
// included: one batch of the workers and the outsider.
require.EqualValues(t, workers+1, w.res.Batch.Size, "worker %d is in the batch of the denied request", w.agent)
}
files := strings.Split(s.git(t, "ls-tree", "-r", "--name-only", "main", "early"), "\n")
require.Len(t, files, workers, "every worker change is on main")
require.NotContains(t, files, "early/"+outsiderName+".txt", "the denied change is not on main")
}