Skip to content

Git designed for AI agents.

14.6 KB
Raw Blame
// Package verify runs the test command on the tip of a branch after lands and
// records the last green tip in a green ref.
//
// The verifier follows the changeset log of the repository from a persisted
// cursor. Lands on the watched branch start a batch; after the batch window
// the current tip is materialized in a work tree (sparse, see Config.Sparse)
// and the test command runs in it. A passing tip moves the green ref, with a
// compare-and-swap, so the green ref always names a tip that passed. A failing
// tip is handed to a Bisector, which finds the culprit and may revert it.
//
// A Verifier is not safe for concurrent use: Run and RunOnce must not run at
// the same time, and a second call fails while the first one is running.
//
// Every test run starts from a work tree that holds exactly the commit under
// test: the tree is removed and cloned again before each run, so a test that
// leaves files, directories or a .git directory behind cannot affect a later
// commit or a bisect step. The cost is one clone per run.
//
// Trust model: the test command runs the code that agents landed, with the
// privileges of the verifier. It inherits the environment of the verifier
// (os.Environ plus Test.Env), so it can read the operator token (token_file or
// the configuration). Operators must run graft-verify as a separate user or in
// a container, with the token file not readable by that user, and should pass
// only the environment the test command needs in Test.Env. Two verifiers must
// not share a work_dir.
package verify

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"log/slog"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/feder-cr/git-ai/internal/graft/workspace"
)

// changesetPage is the number of changesets listed per call by RunOnce.
const changesetPage = 1000

// ErrBusy is returned when Run or RunOnce is called while the verifier is
// already running.
var ErrBusy = errors.New("graft-verify: verifier is already running")

// Tester materializes a commit and runs the test command on it. The bisector
// uses it to test commits between the last green tip and the failing tip.
type Tester interface {
	Test(ctx context.Context, commit string) (TestResult, error)
}

// Failure describes a failing tip that the bisector must analyze.
type Failure struct {
	// Branch is the verified branch, without the refs/heads/ prefix.
	Branch string
	// Good is the last green tip, or "" when no tip is known to be green.
	Good string
	// Bad is the tip that failed the test command.
	Bad string
	// Output is the tail of the output of the failing run.
	Output string
	// Tester tests any commit of the repository.
	Tester Tester
	// Backend reads the repository (refs, trees, changesets).
	Backend workspace.Backend
}

// Bisector analyzes a failing tip. It finds the culprit, may revert it and
// reports what it did. An error is recorded in the outcome of the cycle, and
// the cycle still counts as done. Implementations must be safe to call from a
// single goroutine.
type Bisector interface {
	Bisect(ctx context.Context, f Failure) error
}

// Outcome is the result of one verification cycle.
type Outcome struct {
	Branch string `json:"branch"`
	// Tip is the commit that was tested. It is empty when the cycle was
	// skipped.
	Tip string `json:"tip,omitempty"`
	// Passed reports whether the test command passed on Tip.
	Passed   bool `json:"passed"`
	TimedOut bool `json:"timed_out,omitempty"`
	// Green is the green tip after the cycle.
	Green string `json:"green,omitempty"`
	// GreenMoved reports whether the cycle moved the green ref.
	GreenMoved bool `json:"green_moved"`
	// Skipped is the reason the cycle tested nothing, for example that the
	// branch no longer exists.
	Skipped string `json:"skipped,omitempty"`
	// Output is the tail of the output of the test command.
	Output string `json:"output,omitempty"`
	// BisectError is the error of the bisector, when it failed.
	BisectError string `json:"bisect_error,omitempty"`
	// Cursor is the changeset sequence covered by the cycle.
	Cursor uint64 `json:"cursor"`
}

// Verifier verifies one branch of one repository.
type Verifier struct {
	cfg      Config
	branch   string // refs/heads/<branch>
	backend  workspace.Backend
	pin      *pinnedBackend
	greens   greenRefs
	bisector Bisector
	log      *slog.Logger
	treeDir  string

	// running guards Run and RunOnce.
	running sync.Mutex

	// cursor is the changeset sequence persisted in the state file.
	cursor uint64
	// green is the last known green tip.
	green string
}

// newVerifier creates a verifier over backend and greens. It prepares the work
// directory, loads the state file and reads the current green tip.
func newVerifier(ctx context.Context, cfg Config, backend workspace.Backend, greens greenRefs, logger *slog.Logger) (*Verifier, error) {
	if err := cfg.Validate(); err != nil {
		return nil, err
	}
	if logger == nil {
		logger = slog.Default()
	}

	workDir, err := filepath.Abs(cfg.WorkDir)
	if err != nil {
		return nil, err
	}
	if err := prepareWorkDir(workDir); err != nil {
		return nil, err
	}
	st, err := loadState(cfg.StateFile)
	if err != nil {
		return nil, err
	}
	green, err := greens.Current(ctx, cfg.GreenRef)
	if err != nil {
		return nil, fmt.Errorf("read green ref %s: %w", cfg.GreenRef, err)
	}

	return &Verifier{
		cfg:     cfg,
		branch:  "refs/heads/" + cfg.Branch,
		backend: backend,
		pin:     &pinnedBackend{Backend: backend},
		greens:  greens,
		log:     logger.With("branch", cfg.Branch),
		treeDir: filepath.Join(workDir, treeDirName),
		cursor:  st.Cursor,
		green:   green,
	}, nil
}

// prepareWorkDir creates dir and marks it as a graft-verify work directory. An
// existing directory is accepted only when it is empty or already marked, so
// that a misconfigured work_dir never gets its content removed.
func prepareWorkDir(dir string) error {
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("create work_dir: %w", err)
	}

	marker := filepath.Join(dir, markerName)
	_, err := os.Stat(marker)
	switch {
	case err == nil:
		return nil
	case !errors.Is(err, fs.ErrNotExist):
		return err
	}

	entries, err := os.ReadDir(dir)
	if err != nil {
		return err
	}
	if len(entries) > 0 {
		return fmt.Errorf("work_dir %s is not empty and not a graft-verify work directory", dir)
	}
	return os.WriteFile(marker, []byte("graft-verify work directory; it is removed and rebuilt as needed\n"), 0o644)
}

// SetBisector sets the bisector that analyzes failing tips. Without one, a
// failing tip is only reported.
func (v *Verifier) SetBisector(b Bisector) {
	v.bisector = b
}

// Cursor returns the changeset sequence that the verifier has covered.
func (v *Verifier) Cursor() uint64 {
	return v.cursor
}

// Green returns the last green tip known to the verifier.
func (v *Verifier) Green() string {
	return v.green
}

// Test materializes commit in the work tree and runs the test command on it.
// It implements Tester.
func (v *Verifier) Test(ctx context.Context, commit string) (TestResult, error) {
	// A commit that does not exist must never be tested: the tree of a missing
	// commit could come out empty, and an empty tree passes.
	oid, err := v.backend.ResolveRevision(ctx, commit)
	if err != nil {
		return TestResult{}, fmt.Errorf("resolve %s: %w", commit, err)
	}
	if oid != commit {
		return TestResult{}, fmt.Errorf("resolve %s: got %s, want the full object ID", commit, oid)
	}

	if err := v.materialize(ctx, commit); err != nil {
		return TestResult{}, fmt.Errorf("materialize %s: %w", commit, err)
	}
	return runTests(ctx, v.treeDir, v.cfg.Test, commit)
}

// materialize makes the work tree hold exactly commit. The tree is removed and
// cloned again on every call, and not updated in place: a test run can leave
// anything behind (modified or deleted tracked files, untracked files, empty
// directories, a .git directory, special files) and none of it may reach the
// tree of a later commit or a bisect step. Removing the tree covers all of
// them at once, and no check of the tree can miss a kind of leftover.
func (v *Verifier) materialize(ctx context.Context, commit string) error {
	v.pin.pin(commit)

	if err := removeTree(v.treeDir); err != nil {
		return fmt.Errorf("remove work tree: %w", err)
	}

	if _, err := workspace.Clone(ctx, v.treeDir, v.pin, workspace.CloneOptions{
		Address: v.cfg.Address,
		Storage: v.cfg.Storage,
		Repo:    v.cfg.Repository,
		Branch:  pinRevision,
		Sparse:  v.cfg.Sparse,
	}); err != nil {
		return fmt.Errorf("clone work tree: %w", err)
	}
	return nil
}

// removeTree removes dir and everything in it. A test run may have made
// directories unreadable or unwritable, for example with chmod 000, so every
// directory is made accessible to its owner before its entries are read.
func removeTree(dir string) error {
	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return os.Chmod(path, 0o700)
		}
		return nil
	})
	if err != nil && !errors.Is(err, fs.ErrNotExist) {
		return err
	}
	return os.RemoveAll(dir)
}

// cycle tests the current tip of the branch, which covers every changeset up
// to seq, and acts on the result. An error means that nothing was decided:
// the cursor must not move past seq.
func (v *Verifier) cycle(ctx context.Context, seq uint64) (Outcome, error) {
	out := Outcome{Branch: v.cfg.Branch, Cursor: seq}

	tip, err := v.backend.ResolveRevision(ctx, v.branch)
	if err != nil {
		if errors.Is(err, workspace.ErrNotFound) {
			out.Skipped = "branch does not exist"
			out.Green = v.green
			return out, nil
		}
		return out, fmt.Errorf("resolve branch: %w", err)
	}
	out.Tip = tip

	res, err := v.Test(ctx, tip)
	if err != nil {
		return out, err
	}
	out.Passed = res.Passed
	out.TimedOut = res.TimedOut
	out.Output = res.Output

	if res.Passed {
		moved, err := v.advanceGreen(ctx, tip)
		if err != nil {
			return out, err
		}
		out.GreenMoved = moved
	} else if v.bisector != nil {
		err := v.bisector.Bisect(ctx, Failure{
			Branch:  v.cfg.Branch,
			Good:    v.green,
			Bad:     tip,
			Output:  res.Output,
			Tester:  v,
			Backend: v.backend,
		})
		if err != nil {
			if ctx.Err() != nil {
				return out, ctx.Err()
			}
			out.BisectError = err.Error()
		}
	}

	out.Green = v.green
	return out, nil
}

// advanceGreen moves the green ref to tip, when it is not there already. It
// reports whether the ref moved.
func (v *Verifier) advanceGreen(ctx context.Context, tip string) (bool, error) {
	current, err := v.greens.Current(ctx, v.cfg.GreenRef)
	if err != nil {
		return false, fmt.Errorf("read green ref: %w", err)
	}
	if current == tip {
		v.green = tip
		return false, nil
	}
	if err := v.greens.Move(ctx, v.cfg.GreenRef, current, tip); err != nil {
		return false, fmt.Errorf("move green ref: %w", err)
	}
	v.green = tip
	return true, nil
}

// commit persists seq as the cursor.
func (v *Verifier) commit(seq uint64) error {
	if err := saveState(v.cfg.StateFile, State{Cursor: seq}); err != nil {
		return err
	}
	v.cursor = seq
	return nil
}

// Run follows the changeset log and verifies the branch after lands, until ctx
// is done. It returns nil when ctx is done and an error when the watch of the
// log fails for good. A cycle that fails, for example because the test command
// cannot be started, is retried after the batch window.
func (v *Verifier) Run(ctx context.Context) error {
	if !v.running.TryLock() {
		return ErrBusy
	}
	defer v.running.Unlock()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	window := v.cfg.BatchWindow.Std()
	retryDelay := max(window, time.Second)

	changes, errs := v.backend.WatchChangesets(ctx, v.cursor)
	lastSeq := v.cursor
	pending := false

	timer := time.NewTimer(time.Hour)
	timer.Stop()
	var fire <-chan time.Time
	arm := func(d time.Duration) {
		timer.Reset(d)
		fire = timer.C
	}

	for {
		select {
		case <-ctx.Done():
			return nil

		case c, ok := <-changes:
			if !ok {
				if ctx.Err() != nil {
					return nil
				}
				// errs may already have been observed as closed, in which case
				// it is nil and must not be received from again.
				if errs != nil {
					if err, ok := <-errs; ok && err != nil {
						return fmt.Errorf("watch changesets: %w", err)
					}
				}
				return errors.New("watch changesets: the watch ended")
			}
			if c.Sequence > lastSeq {
				lastSeq = c.Sequence
			}

			if c.Reference == v.branch {
				if !pending {
					pending = true
					arm(window)
				}
			} else if !pending && lastSeq > v.cursor {
				// Nothing on the branch is waiting for a test, so the changes
				// that are not on the branch are covered as well.
				if err := v.commit(lastSeq); err != nil {
					v.log.Error("persist cursor", "err", err)
				}
			}

		case err, ok := <-errs:
			if !ok {
				errs = nil
				continue
			}
			if err != nil {
				return fmt.Errorf("watch changesets: %w", err)
			}

		case <-fire:
			fire = nil
			pending = false

			seq := lastSeq
			out, err := v.cycle(ctx, seq)
			if err != nil {
				if ctx.Err() != nil {
					return nil
				}
				v.log.Error("verification cycle failed, retrying", "err", err, "retry_in", retryDelay)
				pending = true
				arm(retryDelay)
				continue
			}
			if err := v.commit(seq); err != nil {
				return fmt.Errorf("persist cursor: %w", err)
			}
			v.logOutcome(out)
		}
	}
}

// RunOnce verifies the branch once, for CI. It reads the changesets after the
// cursor and, when the branch changed, tests its tip. ran reports whether a
// cycle ran. Without a change on the branch, nothing is tested.
func (v *Verifier) RunOnce(ctx context.Context) (out Outcome, ran bool, err error) {
	if !v.running.TryLock() {
		return Outcome{}, false, ErrBusy
	}
	defer v.running.Unlock()

	seq := v.cursor
	branchChanged := false
	for {
		batch, err := v.backend.Changesets(ctx, seq, changesetPage)
		if err != nil {
			return Outcome{}, false, fmt.Errorf("list changesets: %w", err)
		}
		for _, c := range batch {
			if c.Reference == v.branch {
				branchChanged = true
			}
			seq = c.Sequence
		}
		if len(batch) < changesetPage {
			break
		}
	}

	if !branchChanged {
		if seq > v.cursor {
			if err := v.commit(seq); err != nil {
				return Outcome{}, false, err
			}
		}
		return Outcome{}, false, nil
	}

	out, err = v.cycle(ctx, seq)
	if err != nil {
		return out, true, err
	}
	if err := v.commit(seq); err != nil {
		return out, true, fmt.Errorf("persist cursor: %w", err)
	}
	v.logOutcome(out)
	return out, true, nil
}

// logOutcome logs the result of a cycle.
func (v *Verifier) logOutcome(out Outcome) {
	if out.Skipped != "" {
		v.log.Info("verification skipped", "reason", out.Skipped)
		return
	}
	v.log.Info("verification cycle done",
		"tip", out.Tip,
		"passed", out.Passed,
		"timed_out", out.TimedOut,
		"green", out.Green,
		"green_moved", out.GreenMoved,
		"bisect_error", out.BisectError,
		"cursor", out.Cursor,
	)
}