Skip to content

Git designed for AI agents.

11.8 KB
Raw Blame
// Command graft-bench measures landing throughput and latency of a Graft
// server under concurrent agents. Each agent commits changes on private
// branches and lands them with UserLand on a shared target branch (one by
// default, see --branches). The conflict rate controls how many changes touch
// a shared region of one file. See doc/graft/BENCH.md.
//
// Example:
//
//	graft-bench --address unix:/run/gitaly.sock --repo bench.git \
//	    --agents 50 --changes 10 --conflict-rate 0.2 --mode land
//
// Modes land, land-resolve, land-batch and queue differ in how the changes
// are landed (see Mode). --branches spreads the agents over several branches.
//
// Multi-node runs spread the repositories over several servers, one server per
// address, and report every node as well as the whole run:
//
//	graft-bench --addrs unix:/run/node1.sock,unix:/run/node2.sock,unix:/run/node3.sock \
//	    --repo bench.git --repos 6 --agents 60 --create-repo --token T
//
// Output is a table by default; --format json prints one JSON object. The
// same result can be saved as JSON (--json-out) or as a Markdown report
// (--md-out). Saved JSON results are compared side by side with:
//
//	graft-bench compare --format markdown land.json land-batch.json
package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/feder-cr/git-ai/sdk"
)

// options are the parsed command-line options.
type options struct {
	address    string
	token      string
	agentToken string
	storage    string
	repo       string
	createRepo bool
	format     string
	label      string
	jsonOut    string
	mdOut      string
	timeout    time.Duration
	config     Config
	repos      int
	placement  placement
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	os.Exit(run(ctx, os.Args[1:], os.Stdout, os.Stderr))
}

// run executes the benchmark and returns the process exit code.
func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
	if len(args) > 0 && args[0] == "compare" {
		return runCompare(args[1:], stdout, stderr)
	}

	opts, err := parseOptions(args, stderr)
	if err != nil {
		if errors.Is(err, flag.ErrHelp) {
			return 0
		}
		fmt.Fprintf(stderr, "graft-bench: %v\n", err)
		return 2
	}

	ctx, cancel := context.WithTimeout(ctx, opts.timeout)
	defer cancel()

	if opts.placement.multi {
		return runMultiNode(ctx, opts, stdout, stderr)
	}

	if opts.createRepo {
		if err := createRepository(ctx, opts.address, opts.token, opts.storage, opts.repo); err != nil {
			fmt.Fprintf(stderr, "graft-bench: create repository %s: %v\n", opts.repo, err)
			return 1
		}
	}

	dialOpts := []sdk.DialOption{sdk.WithStorage(opts.storage)}
	switch {
	case opts.token != "":
		dialOpts = append(dialOpts, sdk.WithToken(opts.token))
	case opts.agentToken != "":
		dialOpts = append(dialOpts, sdk.WithAgentToken(opts.agentToken))
	}

	client, err := sdk.Dial(ctx, opts.address, dialOpts...)
	if err != nil {
		fmt.Fprintf(stderr, "graft-bench: dial %s: %v\n", opts.address, err)
		return 1
	}
	defer func() { _ = client.Close() }()

	res, runErr := Run(ctx, opts.config, NewSDKBackend(client, opts.repo))
	if runErr != nil && res.Changes == 0 {
		fmt.Fprintf(stderr, "graft-bench: %v\n", runErr)
		return 1
	}
	res.Label = opts.label

	// A truncated run still prints what it measured, but it is a failure.
	if err := writeResult(stdout, opts.format, res); err != nil {
		fmt.Fprintf(stderr, "graft-bench: write output: %v\n", err)
		return 1
	}
	if err := writeReportFiles(res, opts.jsonOut, opts.mdOut); err != nil {
		fmt.Fprintf(stderr, "graft-bench: %v\n", err)
		return 1
	}
	if runErr != nil {
		fmt.Fprintf(stderr, "graft-bench: %v\n", runErr)
		return 1
	}

	return 0
}

// parseOptions parses args. Usage goes to stderr.
func parseOptions(args []string, stderr io.Writer) (options, error) {
	var opts options

	fs := flag.NewFlagSet("graft-bench", flag.ContinueOnError)
	fs.SetOutput(stderr)
	fs.StringVar(&opts.address, "address", "", "server address: unix:<path>, tcp://<host:port> or tls://<host:port> (required)")
	fs.StringVar(&opts.token, "token", "", "shared token (default $GRAFT_TOKEN)")
	fs.StringVar(&opts.agentToken, "agent-token", "", "agent token (default $GRAFT_AGENT_TOKEN)")
	fs.StringVar(&opts.storage, "storage", sdk.DefaultStorage, "storage name")
	fs.StringVar(&opts.repo, "repo", "", "relative path of the repository (required; with --repos, the base name of the repositories)")
	addrs := fs.String("addrs", "", "comma-separated server addresses, one Gitaly per address (replaces --address; repositories are placed round robin)")
	fs.IntVar(&opts.repos, "repos", 0, "number of repositories, placed round robin over the nodes and worked on round robin by the agents (default: one per address)")
	fs.BoolVar(&opts.createRepo, "create-repo", false, "create the repository first, through the server's RepositoryService (needs --token)")
	fs.StringVar(&opts.config.Branch, "branch", DefaultBranch, "branch the changes land on (with --branches > 1, the prefix of the branch names)")
	fs.IntVar(&opts.config.Branches, "branches", 1, "number of branches the agents are spread over, round robin")
	fs.IntVar(&opts.config.Agents, "agents", DefaultAgents, "concurrent agents (the benchmark targets 50 and 100)")
	fs.IntVar(&opts.config.ChangesPerAgent, "changes", DefaultChangesPerAgent, "changes per agent")
	fs.Float64Var(&opts.config.ConflictRate, "conflict-rate", 0, "fraction of changes that touch the shared region (0 to 1)")
	mode := fs.String("mode", string(ModeLand), "landing mode: land, land-resolve, land-batch or queue")
	fs.DurationVar(&opts.config.TestDuration, "test-duration", 0, "simulated test time per change (queue mode)")
	fs.Int64Var(&opts.config.Seed, "seed", 1, "seed of the change plan")
	maxAttempts := fs.Int("max-attempts", 0, "server-side compare-and-swap attempts per land (0 = server default)")
	fs.StringVar(&opts.format, "format", "table", "output format: table or json")
	fs.StringVar(&opts.label, "label", "", "name of the run in reports, such as batching-on-50 (no | or newlines)")
	fs.StringVar(&opts.jsonOut, "json-out", "", "also write the result as JSON to this file")
	fs.StringVar(&opts.mdOut, "md-out", "", "also write the result as a Markdown report to this file")
	fs.DurationVar(&opts.timeout, "timeout", 10*time.Minute, "overall timeout of the run")

	if err := fs.Parse(args); err != nil {
		return options{}, err
	}
	if fs.NArg() > 0 {
		return options{}, fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " "))
	}

	explicit := map[string]bool{}
	fs.Visit(func(f *flag.Flag) { explicit[f.Name] = true })
	token, agentToken, err := resolveTokens(explicit, opts.token, opts.agentToken,
		os.Getenv("GRAFT_TOKEN"), os.Getenv("GRAFT_AGENT_TOKEN"))
	if err != nil {
		return options{}, err
	}
	opts.token, opts.agentToken = token, agentToken

	opts.config.Mode = Mode(*mode)
	opts.config.MaxAttempts = int32(*maxAttempts)

	place, err := resolvePlacement(opts.address, *addrs, opts.repos, opts.config.Agents)
	if err != nil {
		return options{}, err
	}
	opts.placement = place
	if place.multi {
		opts.config.Repos = place.repos
	}

	switch {
	case opts.repo == "":
		return options{}, errors.New("--repo is required")
	case opts.createRepo && opts.token == "":
		return options{}, errors.New("--create-repo needs the shared --token")
	case opts.format != "table" && opts.format != "json":
		return options{}, fmt.Errorf("unknown format %q, want table or json", opts.format)
	case strings.ContainsAny(opts.label, "|\r\n"):
		return options{}, errors.New("--label must not contain | or newlines")
	case opts.timeout <= 0:
		return options{}, errors.New("--timeout must be positive")
	}

	if err := opts.config.Validate(); err != nil {
		return options{}, err
	}

	return opts, nil
}

// resolveTokens picks the credentials of the run. An explicit flag wins over
// its environment variable, so one flag never conflicts with the other
// variable. Without flags, the shared token of $GRAFT_TOKEN is used in
// preference to $GRAFT_AGENT_TOKEN when both are set. Passing both flags is an
// error.
func resolveTokens(explicit map[string]bool, flagToken, flagAgentToken, envToken, envAgentToken string) (string, string, error) {
	tokenSet, agentSet := explicit["token"], explicit["agent-token"]
	switch {
	case tokenSet && agentSet:
		if flagToken != "" && flagAgentToken != "" {
			return "", "", errors.New("--token and --agent-token are mutually exclusive")
		}
		return flagToken, flagAgentToken, nil
	case tokenSet:
		return flagToken, "", nil
	case agentSet:
		return "", flagAgentToken, nil
	case envToken != "":
		return envToken, "", nil
	default:
		return "", envAgentToken, nil
	}
}

// writeResult prints res as a table or as one JSON object.
func writeResult(w io.Writer, format string, res Result) error {
	if format == "json" {
		enc := json.NewEncoder(w)
		enc.SetIndent("", "  ")
		return enc.Encode(res)
	}

	tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
	rows := [][2]string{
		{"label", res.Label},
		{"mode", string(res.Mode)},
		{"agents", fmt.Sprint(res.Agents)},
		{"branches", fmt.Sprint(res.Branches)},
		{"changes", fmt.Sprintf("%d (%d per agent, conflict rate %.2f)", res.Changes, res.ChangesPerAgent, res.ConflictRate)},
		{"landed", fmt.Sprintf("%d (%d already landed)", res.Landed, res.AlreadyLanded)},
		{"failed", fmt.Sprintf("%d (%d conflicts)", res.Failed, res.Conflicts)},
		{"observed conflicts", fmt.Sprintf("%.3f", res.ObservedConflictRate)},
		{"resolutions", fmt.Sprint(res.Resolutions)},
		{"attempts", fmt.Sprintf("%d over all changes (%.2f per landed change, max %d)", res.Attempts, res.AttemptsPerLand, res.MaxAttempts)},
		{"attempts histogram", formatHistogram(res.AttemptsHistogram) + " (landed changes)"},
		{"failed attempts histogram", formatHistogram(res.FailedAttemptsHistogram) + " (failed changes)"},
		{"unavailable retries", fmt.Sprintf("%d (client retries of Unavailable, landed changes)", res.UnavailableRetries)},
		{"batch size", fmt.Sprintf("mean %.2f, max %d", res.BatchSizeMean, res.BatchSizeMax)},
		{"batch size histogram", formatHistogram(res.BatchSizeHistogram)},
		{"duration", fmt.Sprintf("%.2fs", res.DurationSeconds)},
		{"lands/min", fmt.Sprintf("%.1f", res.LandsPerMinute)},
		{"latency p50", fmt.Sprintf("%.1fms (landed changes)", res.P50MS)},
		{"latency p95", fmt.Sprintf("%.1fms (landed changes)", res.P95MS)},
		{"latency p99", fmt.Sprintf("%.1fms (landed changes)", res.P99MS)},
		{"failed p95", fmt.Sprintf("%.1fms (failed changes)", res.FailedP95MS)},
		{"UserCommitFiles", formatRPCLatency(res.RPCs[rpcCommitFiles])},
		{"UserLand", formatRPCLatency(res.RPCs[rpcLand])},
	}
	for _, row := range rows {
		if row[0] == "label" && row[1] == "" {
			continue
		}
		fmt.Fprintf(tw, "%s\t%s\n", row[0], row[1])
	}
	writeNodeRows(tw, res)
	for _, class := range sortedKeys(res.FailureClasses) {
		fmt.Fprintf(tw, "failure\t%d x %s\n", res.FailureClasses[class], class)
	}
	for _, e := range res.Errors {
		fmt.Fprintf(tw, "error\t%s\n", e)
	}

	return tw.Flush()
}

// writeReportFiles saves res as JSON to jsonPath and as a Markdown report to
// mdPath. An empty path writes nothing.
func writeReportFiles(res Result, jsonPath, mdPath string) error {
	if jsonPath != "" {
		if err := writeFile(jsonPath, func(w io.Writer) error {
			return writeResult(w, "json", res)
		}); err != nil {
			return fmt.Errorf("write --json-out: %w", err)
		}
	}
	if mdPath != "" {
		if err := writeFile(mdPath, func(w io.Writer) error {
			return writeMarkdownReport(w, res)
		}); err != nil {
			return fmt.Errorf("write --md-out: %w", err)
		}
	}

	return nil
}

// writeFile creates path and fills it with write.
func writeFile(path string, write func(io.Writer) error) (err error) {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		if cerr := f.Close(); err == nil {
			err = cerr
		}
	}()

	bw := bufio.NewWriter(f)
	if err := write(bw); err != nil {
		return err
	}

	return bw.Flush()
}