Skip to content

Git designed for AI agents.

13.7 KB
Raw Blame
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

// branchBackend lands changes in memory, per target branch: the first shared
// change of a branch lands, and every later shared change of that branch
// conflicts. Changes that do not share the region always land. With batch set,
// every landed change reports a batch of four.
type branchBackend struct {
	mu       sync.Mutex
	prepared []string
	sharedOn map[string]bool
	landedOn map[string]int
	batch    bool
}

func newBranchBackend() *branchBackend {
	return &branchBackend{sharedOn: map[string]bool{}, landedOn: map[string]int{}}
}

func (b *branchBackend) Prepare(_ context.Context, cfg Config) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	b.prepared = append(b.prepared, cfg.Branch)
	return "base-" + cfg.Branch, nil
}

func (b *branchBackend) Commit(_ context.Context, c Change, base string) (string, error) {
	return base + "-" + c.Path(), nil
}

func (b *branchBackend) Land(_ context.Context, _ Config, c Change, commit string) (LandOutcome, error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	if !strings.HasPrefix(commit, "base-"+c.Target+"-") {
		return LandOutcome{}, errors.New("commit is not based on " + c.Target)
	}
	b.landedOn[c.Target]++

	if c.Shared && b.sharedOn[c.Target] {
		return LandOutcome{Attempts: 1, Conflict: true}, errors.New("merge conflict in " + sharedFilePath)
	}

	out := LandOutcome{Attempts: 1}
	if c.Shared {
		b.sharedOn[c.Target] = true
		out.Attempts = 2
	}
	if b.batch {
		out.BatchSize = 4
		out.BatchPosition = c.Seq%4 + 1
	}

	return out, nil
}

func TestConfigBranchNames(t *testing.T) {
	t.Parallel()

	t.Run("several branches", func(t *testing.T) {
		t.Parallel()

		cfg := Config{Agents: 10, Branch: "bench", Branches: 3}
		require.Equal(t, 3, cfg.BranchCount())
		require.Equal(t, "bench-00", cfg.BranchName(0))
		require.Equal(t, "bench-02", cfg.BranchName(2))
		require.Equal(t, "bench-01", cfg.TargetOf(4), "agents are spread round robin")
		require.Equal(t, "bench-00", cfg.TargetOf(9))
	})

	t.Run("branches of the agents of each repository", func(t *testing.T) {
		t.Parallel()

		// Agent a works on repository a % 2. Spreading by agent alone would put
		// every agent of repository 0 on bench-00 and every agent of repository 1
		// on bench-01.
		cfg := Config{Agents: 8, Branch: "bench", Branches: 2, Repos: 2}
		var targets []string
		for agent := 0; agent < cfg.Agents; agent++ {
			targets = append(targets, cfg.TargetOf(agent))
		}
		require.Equal(t, []string{"bench-00", "bench-00", "bench-01", "bench-01", "bench-00", "bench-00", "bench-01", "bench-01"}, targets)
	})

	t.Run("zero branches means one", func(t *testing.T) {
		t.Parallel()

		cfg := Config{Agents: 10, Branch: "bench"}
		require.Equal(t, 1, cfg.BranchCount())
		require.Equal(t, "bench", cfg.BranchName(0))
		require.Equal(t, "bench", cfg.TargetOf(9))
	})
}

func TestConfigValidateBatchModeAndBranches(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name    string
		modify  func(*Config)
		wantErr string
	}{
		{"land-batch is a mode", func(c *Config) { c.Mode = ModeLandBatch }, ""},
		{"branches equal agents", func(c *Config) { c.Branches = c.Agents }, ""},
		{"negative branches", func(c *Config) { c.Branches = -1 }, "branches must not be negative"},
		{"more branches than agents", func(c *Config) { c.Branches = c.Agents + 1 }, "must not exceed agents"},
		{"negative repos", func(c *Config) { c.Repos = -1 }, "repos must not be negative"},
		{"every repository on every branch", func(c *Config) {
			c.Repos = 2
			c.Branches = 2
		}, ""},
		{"a repository without an agent on a branch", func(c *Config) {
			c.Repos = 3
			c.Branches = 2
		}, "every repository needs an agent on every branch"},
		{"test duration outside queue mode", func(c *Config) {
			c.Mode = ModeLandBatch
			c.TestDuration = time.Millisecond
		}, "only used in queue mode"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			cfg := baseConfig()
			tt.modify(&cfg)
			err := cfg.Validate()
			if tt.wantErr == "" {
				require.NoError(t, err)
				return
			}
			require.ErrorContains(t, err, tt.wantErr)
		})
	}
}

func TestRunSpreadsAgentsOverBranches(t *testing.T) {
	t.Parallel()

	cfg := Config{
		Agents:          6,
		ChangesPerAgent: 2,
		ConflictRate:    1,
		Mode:            ModeLand,
		Branch:          DefaultBranch,
		Branches:        3,
		Seed:            1,
	}
	be := newBranchBackend()
	res, err := Run(context.Background(), cfg, be)
	require.NoError(t, err)

	require.Equal(t, []string{"bench-00", "bench-01", "bench-02"}, be.prepared)
	require.Equal(t, map[string]int{"bench-00": 4, "bench-01": 4, "bench-02": 4}, be.landedOn)

	require.Equal(t, 3, res.Branches)
	require.Equal(t, 12, res.Changes)
	require.Equal(t, 3, res.Landed, "one change per branch lands")
	require.Equal(t, 9, res.Failed)
	require.Equal(t, 9, res.Conflicts, "every failure is a conflict, so no change landed on the wrong base")
	require.InDelta(t, 0.75, res.ObservedConflictRate, 1e-9)
	require.Equal(t, int64(15), res.Attempts, "3 landed changes of 2 attempts and 9 conflicts of 1")
	require.InDelta(t, 5, res.AttemptsPerLand, 1e-9)
	require.Equal(t, int32(2), res.MaxAttempts)
}

func TestRunHundredAgentsInBatchMode(t *testing.T) {
	t.Parallel()

	require.Equal(t, 50, DefaultAgents)

	cfg := baseConfig()
	cfg.Agents = 100
	cfg.ChangesPerAgent = 2
	cfg.Mode = ModeLandBatch

	be := newBranchBackend()
	be.batch = true
	res, err := Run(context.Background(), cfg, be)
	require.NoError(t, err)

	require.Equal(t, 200, res.Changes)
	require.Equal(t, 200, res.Landed)
	require.Zero(t, res.Failed)
	require.Zero(t, res.FailedP95MS)
	require.InDelta(t, 1, res.AttemptsPerLand, 1e-9)
	require.InDelta(t, 4, res.BatchSizeMean, 1e-9)
	require.Equal(t, 4, res.BatchSizeMax)
	require.Equal(t, ModeLandBatch, res.Mode)
}

func TestPercentileNearestRank(t *testing.T) {
	t.Parallel()

	hundred := make([]float64, 0, 100)
	for i := 1; i <= 100; i++ {
		hundred = append(hundred, float64(i))
	}
	ten := hundred[:10]
	twenty := hundred[:20]

	tests := []struct {
		name   string
		values []float64
		p      float64
		want   float64
	}{
		{"p50 of 100", hundred, 50, 50},
		{"p95 of 100", hundred, 95, 95},
		{"p99 of 100", hundred, 99, 99},
		{"p100 of 100", hundred, 100, 100},
		{"p1 of 100", hundred, 1, 1},
		{"p90 of 10", ten, 90, 9},
		{"p95 of 10", ten, 95, 10},
		{"p95 of 20", twenty, 95, 19},
		{"p50 of 3", []float64{1, 2, 3}, 50, 2},
		{"p33 of 3", []float64{1, 2, 3}, 33, 1},
		{"p34 of 3", []float64{1, 2, 3}, 34, 2},
	}

	for _, tt := range tests {
		require.Equal(t, tt.want, percentile(tt.values, tt.p), tt.name)
	}
}

func TestAggregateLatencyCoversLandedChangesOnly(t *testing.T) {
	t.Parallel()

	samples := []Sample{
		{Latency: 10 * time.Millisecond, Landed: true},
		{Latency: 20 * time.Millisecond, Landed: true},
		{Latency: 30 * time.Millisecond, Landed: true},
		{Latency: 1000 * time.Millisecond, Err: errors.New("merge conflict"), Conflict: true},
		{Latency: 2000 * time.Millisecond, Err: errors.New("hook rejected")},
	}

	res := aggregate(baseConfig(), samples, time.Second)

	require.Equal(t, 5, res.Changes)
	require.Equal(t, 3, res.Landed)
	require.Equal(t, 2, res.Failed)
	require.InDelta(t, 0.2, res.ObservedConflictRate, 1e-9)
	require.InDelta(t, 20, res.P50MS, 1e-9, "failed changes are not in the landed percentiles")
	require.InDelta(t, 30, res.P99MS, 1e-9)
	require.InDelta(t, 2000, res.FailedP95MS, 1e-9)
}

func TestAggregateBatchAndAttemptsStats(t *testing.T) {
	t.Parallel()

	t.Run("batches of landed changes", func(t *testing.T) {
		t.Parallel()

		samples := []Sample{
			{Landed: true, Outcome: LandOutcome{Attempts: 1, BatchSize: 2, UnavailableRetries: 2}},
			{Landed: true, Outcome: LandOutcome{Attempts: 3, BatchSize: 4, UnavailableRetries: 1}},
			{Err: errors.New("x"), Outcome: LandOutcome{Attempts: 5, BatchSize: 9}},
		}
		res := aggregate(baseConfig(), samples, time.Second)

		require.Equal(t, 3, res.UnavailableRetries, "the client retries of Unavailable are reported")
		require.InDelta(t, 3, res.BatchSizeMean, 1e-9, "a failed change is not part of a landed batch")
		require.Equal(t, 4, res.BatchSizeMax)
		require.Equal(t, int64(9), res.Attempts)
		require.InDelta(t, 4.5, res.AttemptsPerLand, 1e-9, "attempts of failed changes count as server work, not per landed change")
		require.Equal(t, int32(5), res.MaxAttempts)
	})

	t.Run("nothing landed", func(t *testing.T) {
		t.Parallel()

		samples := []Sample{{Latency: time.Millisecond, Err: errors.New("x")}}
		res := aggregate(baseConfig(), samples, 0)

		require.Zero(t, res.AttemptsPerLand)
		require.Zero(t, res.P50MS)
		require.Zero(t, res.LandsPerMinute)
		require.InDelta(t, 1, res.FailedP95MS, 1e-9)
	})
}

func TestParseOptionsBranchesAndBatchMode(t *testing.T) {
	t.Parallel()

	opts, err := parseOptions([]string{
		"--address", "unix:/tmp/g.sock", "--repo", "r.git",
		"--mode", "land-batch", "--branches", "4", "--agents", "100",
	}, &bytes.Buffer{})
	require.NoError(t, err)
	require.Equal(t, ModeLandBatch, opts.config.Mode)
	require.Equal(t, 4, opts.config.Branches)
	require.Equal(t, 100, opts.config.Agents)
}

func TestWriteComparison(t *testing.T) {
	t.Parallel()

	results := []Result{
		{Mode: ModeLand, Agents: 50, Branches: 1, Changes: 500, Landed: 480, Failed: 20, P95MS: 40, FailedP95MS: 900},
		{Mode: ModeLandBatch, Agents: 50, Branches: 1, Changes: 500, Landed: 500, BatchSizeMean: 6.5, BatchSizeMax: 12, P95MS: 25},
	}
	labels := []string{"land.json", "land-batch.json"}

	t.Run("table", func(t *testing.T) {
		t.Parallel()

		var out bytes.Buffer
		require.NoError(t, writeComparison(&out, "table", labels, results))
		text := out.String()
		require.Contains(t, text, "metric")
		require.Contains(t, text, "land-batch.json")
		require.Contains(t, text, "batch size mean")
		require.Contains(t, text, "6.50")
		require.Contains(t, text, "latency p95 (failed)")
	})

	t.Run("markdown", func(t *testing.T) {
		t.Parallel()

		var out bytes.Buffer
		require.NoError(t, writeComparison(&out, "markdown", labels, results))
		lines := strings.Split(out.String(), "\n")
		require.Equal(t, "| metric | land.json | land-batch.json |", lines[0])
		require.Equal(t, "| --- | --- | --- |", lines[1])
		require.Contains(t, out.String(), "| landed | 480 | 500 |")
	})

	t.Run("errors", func(t *testing.T) {
		t.Parallel()

		var out bytes.Buffer
		require.Error(t, writeComparison(&out, "table", labels[:1], results))
		require.Error(t, writeComparison(&out, "table", nil, nil))
		require.Error(t, writeComparison(&out, "json", labels, results))
	})
}

func TestRunCompareCommand(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	var paths []string
	for _, name := range []string{"land.json", "land-batch.json"} {
		data, err := json.Marshal(Result{Mode: ModeLand, Agents: 50, Branches: 1})
		require.NoError(t, err)
		path := filepath.Join(dir, name)
		require.NoError(t, os.WriteFile(path, data, 0o600))
		paths = append(paths, path)
	}

	var out, errOut bytes.Buffer
	code := run(context.Background(), append([]string{"compare", "--format", "markdown"}, paths...), &out, &errOut)
	require.Zero(t, code, errOut.String())
	require.Contains(t, out.String(), "| metric | land.json | land-batch.json |")

	out.Reset()
	errOut.Reset()
	require.Equal(t, 2, run(context.Background(), []string{"compare"}, &out, &errOut), "compare needs a file")

	errOut.Reset()
	require.Equal(t, 1, run(context.Background(), []string{"compare", filepath.Join(dir, "missing.json")}, &out, &errOut))
	require.Contains(t, errOut.String(), "missing.json")
}

func TestAggregateRPCLatency(t *testing.T) {
	t.Parallel()

	samples := []Sample{
		{Landed: true, Committed: true, CommitLatency: 10 * time.Millisecond, LandLatency: 100 * time.Millisecond},
		{Landed: true, Committed: true, CommitLatency: 20 * time.Millisecond, LandLatency: 200 * time.Millisecond},
		{Committed: true, CommitLatency: 30 * time.Millisecond, LandLatency: 900 * time.Millisecond, Err: errors.New("merge conflict"), Conflict: true},
		{CommitLatency: 500 * time.Millisecond, Err: errors.New("commit: unavailable")},
	}

	res := aggregate(baseConfig(), samples, time.Second)

	commit := res.RPCs[rpcCommitFiles]
	require.Equal(t, 4, commit.Calls)
	require.Equal(t, 1, commit.Failed)
	require.InDelta(t, 20, commit.P50MS, 1e-9, "failed calls are not in the percentiles")
	require.InDelta(t, 30, commit.P99MS, 1e-9)

	land := res.RPCs[rpcLand]
	require.Equal(t, 3, land.Calls, "a change whose commit failed does not land")
	require.Equal(t, 1, land.Failed)
	require.InDelta(t, 100, land.P50MS, 1e-9)
	require.InDelta(t, 200, land.P99MS, 1e-9)
}

// slowBackend takes a fixed time for each call.
type slowBackend struct {
	commit, land time.Duration
}

func (b slowBackend) Prepare(context.Context, Config) (string, error) { return "base", nil }

func (b slowBackend) Commit(context.Context, Change, string) (string, error) {
	time.Sleep(b.commit)
	return "commit", nil
}

func (b slowBackend) Land(context.Context, Config, Change, string) (LandOutcome, error) {
	time.Sleep(b.land)
	return LandOutcome{Attempts: 1}, nil
}

// TestRunMeasuresEachRPC times the commit and the land of a change separately: the latency of the
// change covers both, each RPC only its own call.
func TestRunMeasuresEachRPC(t *testing.T) {
	t.Parallel()

	cfg := baseConfig()
	cfg.Agents, cfg.ChangesPerAgent = 2, 2
	res, err := Run(context.Background(), cfg, slowBackend{commit: 20 * time.Millisecond, land: 60 * time.Millisecond})
	require.NoError(t, err)

	commit, land := res.RPCs[rpcCommitFiles], res.RPCs[rpcLand]
	require.Equal(t, 4, commit.Calls)
	require.Equal(t, 4, land.Calls)
	require.GreaterOrEqual(t, commit.P50MS, 20.0)
	require.Less(t, commit.P50MS, 60.0, "the commit does not include the land")
	require.GreaterOrEqual(t, land.P50MS, 60.0)
	require.GreaterOrEqual(t, res.P50MS, 80.0, "a change takes its commit and its land")
}