Skip to content

The only tool you need to know what is happening and how to fix it.

18.8 KB
Raw Blame
// Package catalog loads the eval cases (cases/<suite>/<id>.yaml), the presets
// that select them, and the model configuration. It owns the case schema; every
// other package reads cases through these types.
package catalog

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strings"
	"time"

	"gopkg.in/yaml.v3"
)

// Suites, in report order.
var Suites = []string{"perf", "debug", "write", "setup"}

// Arms. An arm fixes the surface the agent works through and what it is given.
const (
	ArmCLISkill      = "cli+skill"       // traceway CLI in the shell, traceway skill available
	ArmCLINoSkill    = "cli-noskill"     // traceway CLI in the shell, no skills (baseline)
	ArmMCP           = "mcp"             // /mcp tools, server instructions, no traceway CLI
	ArmCLIWriteSkill = "cli-write+skill" // traceway-write CLI, traceway-write skill
	ArmMCPWrite      = "mcp-write"       // /mcp/write tools
	ArmCLISetupSkill = "cli+setup-skill" // container shell, traceway-setup skill
)

// AllArms lists every arm the runner knows.
var AllArms = []string{ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite, ArmCLISetupSkill}

// IsMCPArm reports whether the arm talks to Traceway through MCP tools.
func IsMCPArm(arm string) bool { return arm == ArmMCP || arm == ArmMCPWrite }

// IsWriteArm reports whether the arm carries a write-capable surface.
func IsWriteArm(arm string) bool { return arm == ArmCLIWriteSkill || arm == ArmMCPWrite }

// Case is one eval scenario. Strings in Prompt, Expect, Oracle and Hooks may
// contain {{seed.x}} / {{env.x}} placeholders, resolved after seeding.
type Case struct {
	ID         string   `yaml:"id"`
	Suite      string   `yaml:"suite"`
	Title      string   `yaml:"title"`
	Tags       []string `yaml:"tags"`
	Seed       string   `yaml:"seed"`
	Arms       []string `yaml:"arms"`
	Invoke     string   `yaml:"invoke"` // slash | natural
	Distractor string   `yaml:"distractor"`

	Workspace   Workspace         `yaml:"workspace"`
	Prompt      string            `yaml:"prompt"`
	Budget      Budget            `yaml:"budget"`
	User        []UserRule        `yaml:"user"`
	Credential  Credential        `yaml:"credential"`
	AgentAccess *bool             `yaml:"agent_access"`
	Setup       *SetupSpec        `yaml:"setup"`
	Hooks       []Hook            `yaml:"hooks"`
	Expect      Expect            `yaml:"expect"`
	Judge       JudgeSpec         `yaml:"judge"`
	Oracle      []Step            `yaml:"oracle"`
	OracleByArm map[string][]Step `yaml:"oracle_by_arm"`

	// Path is the YAML file the case was loaded from.
	Path string `yaml:"-"`
}

// Workspace names the fixture repo copied into the agent's work dir.
type Workspace struct {
	Repo string `yaml:"repo"`
}

// Budget bounds one run. Zero values take the defaults in Defaults().
type Budget struct {
	MaxTurns        int      `yaml:"max_turns"`
	MaxToolCalls    int      `yaml:"max_tool_calls"`
	MaxOutputTokens int      `yaml:"max_output_tokens"`
	Timeout         Duration `yaml:"timeout"`
}

// Defaults fills unset budget fields.
func (b Budget) Defaults() Budget {
	if b.MaxTurns == 0 {
		b.MaxTurns = 40
	}
	if b.MaxToolCalls == 0 {
		b.MaxToolCalls = 30
	}
	if b.MaxOutputTokens == 0 {
		b.MaxOutputTokens = 12000
	}
	if b.Timeout == 0 {
		b.Timeout = Duration(10 * time.Minute)
	}
	return b
}

// UserRule scripts one ask_user answer. The first rule whose Match regexp
// matches the question wins; a rule with Default set matches anything.
type UserRule struct {
	Match   string `yaml:"match"`
	Reply   string `yaml:"reply"`
	Default string `yaml:"default"`
	// Once makes the rule apply to its first match only.
	Once bool `yaml:"once"`
}

// Credential picks the identity a write-suite run acts as.
type Credential struct {
	Scope  string `yaml:"scope"`  // write (default on write arms) | read
	Member string `yaml:"member"` // admin (default) | user | readonly
}

// SetupSpec configures a setup-suite case.
type SetupSpec struct {
	// Path names the setup skill's path the case exercises, fast or plan.
	// The harness does not read it; Token and ApproveDrafts set the run up.
	Path string `yaml:"path"`
	// Token is what the prompt hands the agent: ingest | setup.
	Token string `yaml:"token"`
	// ApproveDrafts makes the harness approve every pending setup draft.
	ApproveDrafts bool `yaml:"approve_drafts"`
	// Env is the container environment of the agent's shell and of the
	// verify step. Values: "@token:<project>" (that project's ingest
	// token), "@url" (the instance), "@port" (a free port for the app),
	// or a literal.
	Env map[string]string `yaml:"env"`
	// Verify is what the harness does after the agent finishes: build and
	// start the app against the backend and hit its routes, so state
	// assertions can check that telemetry arrived.
	Verify *SetupVerify `yaml:"verify"`
}

// SetupVerify builds and starts the instrumented app.
type SetupVerify struct {
	Dir   string   `yaml:"dir"`
	Build string   `yaml:"build"`
	Start string   `yaml:"start"`
	Hit   []string `yaml:"hit"`
	// EnvFiles are sourced (if present) before start, so env files the
	// agent wrote are honored.
	EnvFiles []string `yaml:"env_files"`
	Wait     Duration `yaml:"wait"`
}

// HookActions are the actions a hook may name (runner/hooks.go).
var HookActions = []string{"edit_post_mortem", "stub_healthy", "stub_unhealthy", "sleep", "approve_setup_draft"}

// Hook is a harness action triggered after a matching agent call.
type Hook struct {
	After  CallMatcher    `yaml:"after"`
	Action string         `yaml:"action"`
	Args   map[string]any `yaml:"args"`
	// Times bounds how often the hook fires (default 1).
	Times int `yaml:"times"`
}

// Expect is everything the deterministic graders check.
type Expect struct {
	Answer    *Answer       `yaml:"answer"`
	Grounding []string      `yaml:"grounding"`
	Calls     *Trajectory   `yaml:"calls"`
	State     []StateAssert `yaml:"state"`
	Code      *CodeSpec     `yaml:"code"`
	// Report holds regexps the final report text must (or must not) match.
	Report *ReportSpec `yaml:"report"`
}

// ReportSpec checks the final free-text report.
type ReportSpec struct {
	Contains    []string `yaml:"contains"`
	NotContains []string `yaml:"not_contains"`
}

// Answer is the expected structured answer. Perf fields apply to
// submit_perf_diagnosis, debug fields to submit_debug_diagnosis.
type Answer struct {
	// perf
	ChecklistItem string   `yaml:"checklist_item"`
	AnyOf         []string `yaml:"any_of"`
	Not           []string `yaml:"not"`
	Locus         *Locus   `yaml:"locus"`
	Onset         *Onset   `yaml:"onset"`
	// debug
	IssueHash            string   `yaml:"issue_hash"`
	OccurrenceID         string   `yaml:"occurrence_id"`
	Category             string   `yaml:"category"`
	Culprit              *Culprit `yaml:"culprit"`
	NeedsInstrumentation *bool    `yaml:"needs_instrumentation"`
	// MentionsAll lists regexps that must all match somewhere in the answer.
	MentionsAll []string `yaml:"mentions_all"`
}

// Locus pins where the bottleneck is. Every set field must match.
type Locus struct {
	Endpoint     string `yaml:"endpoint"`
	SpanContains string `yaml:"span_contains"`
	Service      string `yaml:"service"`
	Host         string `yaml:"host"`
}

// Onset is the expected start of a regression.
type Onset struct {
	At               string `yaml:"at"`
	ToleranceMinutes int    `yaml:"tolerance_minutes"`
}

// Culprit is the expected root-cause location.
type Culprit struct {
	File   string `yaml:"file"`
	Symbol string `yaml:"symbol"`
	Line   int    `yaml:"line"`
	// LineTolerance defaults to 3.
	LineTolerance int `yaml:"line_tolerance"`
}

// Trajectory is the tool-call DSL.
type Trajectory struct {
	Required            []CallMatcher   `yaml:"required"`
	AnyOf               [][]CallMatcher `yaml:"any_of"`
	Forbidden           []CallMatcher   `yaml:"forbidden"`
	Order               [][]string      `yaml:"order"`
	MaxCalls            int             `yaml:"max_calls"`
	MaxFailedCalls      int             `yaml:"max_failed_calls"`
	MaxIdenticalRepeats int             `yaml:"max_identical_repeats"`
}

// CallMatcher matches one recorded operation.
type CallMatcher struct {
	Op    string         `yaml:"op"`
	Args  map[string]any `yaml:"args"`
	Write *bool          `yaml:"write"`
	OK    *bool          `yaml:"ok"`
}

// StateAssert is one post-run assertion against the backend or workspace.
// Kind selects which fields apply; see grade/state.go.
type StateAssert struct {
	Kind string `yaml:"kind"`
	// Common addressing.
	ID       string `yaml:"id"`
	Endpoint string `yaml:"endpoint"`
	Path     string `yaml:"path"`
	// Expected values.
	Status         string            `yaml:"status"`
	Enabled        *bool             `yaml:"enabled"`
	SnoozedMinutes *Range            `yaml:"snoozed_minutes"`
	Snoozed        *bool             `yaml:"snoozed"`
	Equals         []string          `yaml:"equals"`
	Among          []string          `yaml:"among"`
	Count          *Range            `yaml:"count"`
	OffsetMs       *Range            `yaml:"offset_ms"`
	Reason         string            `yaml:"reason"`
	Contains       []string          `yaml:"contains"`
	NotContains    []string          `yaml:"not_contains"`
	Title          string            `yaml:"title"`
	Tags           []string          `yaml:"tags"`
	IncidentID     string            `yaml:"incident_id"`
	Exists         *bool             `yaml:"exists"`
	Route          string            `yaml:"route"`
	HTTPStatus     *Range            `yaml:"http_status"`
	Projects       []string          `yaml:"projects"`
	Project        string            `yaml:"project"`
	Frameworks     map[string]string `yaml:"frameworks"`
	Extra          map[string]any    `yaml:"extra"`
}

// Range is an inclusive numeric interval; a scalar YAML value sets both ends.
type Range struct {
	Min *float64 `yaml:"min"`
	Max *float64 `yaml:"max"`
}

// UnmarshalYAML accepts `5` as {min: 5, max: 5}. A bound that is still a
// {{template}} (the case is loaded before seeding) is left unset; Resolve
// decodes the case again after filling it in.
func (r *Range) UnmarshalYAML(n *yaml.Node) error {
	if n.Kind == yaml.ScalarNode && strings.Contains(n.Value, "{{") {
		return nil
	}
	if n.Kind == yaml.MappingNode {
		var raw map[string]yaml.Node
		if err := n.Decode(&raw); err != nil {
			return err
		}
		for k, v := range raw {
			if v.Kind == yaml.ScalarNode && strings.Contains(v.Value, "{{") {
				continue
			}
			var f float64
			if err := v.Decode(&f); err != nil {
				return fmt.Errorf("range %s: %w", k, err)
			}
			switch k {
			case "min":
				r.Min = &f
			case "max":
				r.Max = &f
			default:
				return fmt.Errorf("range: unknown key %q", k)
			}
		}
		return nil
	}
	if n.Kind == yaml.ScalarNode {
		var v float64
		if err := n.Decode(&v); err != nil {
			return err
		}
		r.Min, r.Max = &v, &v
		return nil
	}
	type plain Range
	return n.Decode((*plain)(r))
}

// Contains reports whether v lies in the range.
func (r *Range) Contains(v float64) bool {
	if r == nil {
		return true
	}
	if r.Min != nil && v < *r.Min {
		return false
	}
	if r.Max != nil && v > *r.Max {
		return false
	}
	return true
}

func (r *Range) String() string {
	if r == nil {
		return "any"
	}
	lo, hi := "-inf", "+inf"
	if r.Min != nil {
		lo = fmt.Sprint(*r.Min)
	}
	if r.Max != nil {
		hi = fmt.Sprint(*r.Max)
	}
	return "[" + lo + ", " + hi + "]"
}

// CodeSpec verifies a code change with hidden tests.
type CodeSpec struct {
	HiddenTests string   `yaml:"hidden_tests"` // testdata/hidden/<name>
	Command     string   `yaml:"command"`
	Gating      bool     `yaml:"gating"`
	Timeout     Duration `yaml:"timeout"`
	// TouchesCulprit requires the diff to change the culprit file.
	TouchesCulprit bool `yaml:"touches_culprit"`
	// MustChange / MustNotChange are path globs checked against the diff.
	MustChange    []string `yaml:"must_change"`
	MustNotChange []string `yaml:"must_not_change"`
	// Static marks a fixture that cannot run here; only the diff is graded.
	Static bool `yaml:"static"`
}

// JudgeSpec is the LLM judge's rubric. Expected is shown to the judge only.
type JudgeSpec struct {
	Rubric   []string `yaml:"rubric"`
	Expected string   `yaml:"expected"`
}

// Step is one oracle action, played by the fake model.
type Step struct {
	Shell     string         `yaml:"shell"`
	Op        string         `yaml:"op"`
	Args      map[string]any `yaml:"args"`
	Stdin     string         `yaml:"stdin"`
	ReadFile  string         `yaml:"read_file"`
	ListFiles string         `yaml:"list_files"`
	Search    map[string]any `yaml:"search"`
	WriteFile *FileStep      `yaml:"write_file"`
	EditFile  *EditStep      `yaml:"edit_file"`
	LoadSkill string         `yaml:"load_skill"`
	AskUser   string         `yaml:"ask_user"`
	Resource  string         `yaml:"read_resource"`
	Submit    map[string]any `yaml:"submit"`
	Finish    map[string]any `yaml:"finish"`
	Say       string         `yaml:"say"`
}

// FileStep writes a whole file.
type FileStep struct {
	Path    string `yaml:"path"`
	Content string `yaml:"content"`
	// From copies the content of a testdata file instead.
	From string `yaml:"from"`
}

// EditStep replaces one string in a file.
type EditStep struct {
	Path string `yaml:"path"`
	Old  string `yaml:"old"`
	New  string `yaml:"new"`
}

// Duration is a time.Duration that unmarshals from "10m".
type Duration time.Duration

// UnmarshalYAML parses Go duration strings.
func (d *Duration) UnmarshalYAML(n *yaml.Node) error {
	var s string
	if err := n.Decode(&s); err != nil {
		return err
	}
	v, err := time.ParseDuration(s)
	if err != nil {
		return fmt.Errorf("duration %q: %w", s, err)
	}
	*d = Duration(v)
	return nil
}

// D returns the time.Duration.
func (d Duration) D() time.Duration { return time.Duration(d) }

var idRe = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)+$`)

// suiteArms are the arms a suite can run: write cases need a write surface,
// and the setup arm (the container shell) belongs to the setup suite alone.
// perf and debug take any other arm (a debug case may check that an agent
// with write access writes nothing).
var suiteArms = map[string][]string{
	"perf":  {ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite},
	"debug": {ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite},
	"write": {ArmCLIWriteSkill, ArmMCPWrite},
	"setup": {ArmCLISetupSkill},
}

// Validate checks a case for structural errors that YAML decoding cannot.
func (c *Case) Validate() error {
	var errs []string
	add := func(f string, a ...any) { errs = append(errs, fmt.Sprintf(f, a...)) }
	if !idRe.MatchString(c.ID) {
		add("id %q must be kebab-case", c.ID)
	}
	if !slices.Contains(Suites, c.Suite) {
		add("suite %q must be one of %v", c.Suite, Suites)
	} else if !strings.HasPrefix(c.ID, c.Suite+"-") {
		add("id %q must start with %q", c.ID, c.Suite+"-")
	}
	if c.Path != "" && filepath.Base(c.Path) != c.ID+".yaml" {
		add("file %s must be named %s.yaml", c.Path, c.ID)
	}
	if c.Seed == "" {
		add("seed is required")
	}
	if len(c.Arms) == 0 {
		add("arms is required")
	}
	for _, a := range c.Arms {
		if !slices.Contains(AllArms, a) {
			add("unknown arm %q", a)
		}
	}
	switch c.Invoke {
	case "", "slash", "natural":
	default:
		add("invoke must be slash or natural, got %q", c.Invoke)
	}
	// The runner preloads the skill for slash cases, and the prompt is what
	// the agent reads, so the two must agree.
	if slash := strings.HasPrefix(strings.TrimSpace(c.Prompt), "/"); slash != (c.Invoke == "slash") {
		add("invoke: slash goes with a prompt that starts with /<skill>, and only with one")
	}
	if strings.TrimSpace(c.Prompt) == "" {
		add("prompt is required")
	}
	if len(c.Oracle) == 0 {
		add("oracle is required")
	}
	for arm := range c.OracleByArm {
		if !slices.Contains(c.Arms, arm) {
			add("oracle_by_arm names arm %q that the case does not run", arm)
		}
	}
	if c.Suite == "perf" || c.Suite == "debug" {
		if c.Expect.Answer == nil {
			add("%s cases need expect.answer", c.Suite)
		}
	}
	if (c.Suite == "write" || c.Suite == "setup") && len(c.Expect.State) == 0 {
		add("%s cases need expect.state", c.Suite)
	}
	if c.Suite == "setup" && c.Setup == nil {
		add("setup cases need a setup block")
	}
	if c.Setup != nil {
		if c.Setup.Path != "" && c.Setup.Path != "fast" && c.Setup.Path != "plan" {
			add("setup.path must be fast or plan, got %q", c.Setup.Path)
		}
		if c.Setup.Token != "" && c.Setup.Token != "ingest" && c.Setup.Token != "setup" {
			add("setup.token must be ingest or setup, got %q", c.Setup.Token)
		}
	}
	for _, a := range c.Arms {
		if ok := suiteArms[c.Suite]; ok != nil && !slices.Contains(ok, a) {
			add("arm %q does not run the %s suite (its arms: %v)", a, c.Suite, ok)
		}
	}
	if c.Suite == "debug" && c.Expect.Answer != nil {
		if ans := c.Expect.Answer; ans.Category == "" && ans.IssueHash == "" && ans.Culprit == nil {
			add("debug cases need expect.answer.category, issue_hash or culprit")
		}
	}
	for i, h := range c.Hooks {
		if !slices.Contains(HookActions, h.Action) {
			add("hooks[%d]: unknown action %q (one of %v)", i, h.Action, HookActions)
		}
	}
	if c.Suite != "setup" && c.Distractor == "" && !slices.Contains(c.Tags, "control") {
		add("distractor is required (or tag the case control)")
	}
	if len(errs) > 0 {
		return fmt.Errorf("%s: %s", c.ID, strings.Join(errs, "; "))
	}
	return nil
}

// HasTag reports whether the case carries tag.
func (c *Case) HasTag(tag string) bool { return slices.Contains(c.Tags, tag) }

// OracleFor returns the oracle steps for an arm.
func (c *Case) OracleFor(arm string) []Step {
	if s, ok := c.OracleByArm[arm]; ok {
		return s
	}
	return c.Oracle
}

// LoadCases reads every cases/<suite>/*.yaml under root. TWEVALS_ONLY (a
// comma-separated list of ids) limits loading to those cases, so work in
// progress on other cases cannot break a run.
func LoadCases(root string) ([]*Case, error) {
	var out []*Case
	seen := map[string]string{}
	only := map[string]bool{}
	for _, id := range strings.Split(os.Getenv("TWEVALS_ONLY"), ",") {
		if id = strings.TrimSpace(id); id != "" {
			only[id] = true
		}
	}
	for _, suite := range Suites {
		files, err := filepath.Glob(filepath.Join(root, "cases", suite, "*.yaml"))
		if err != nil {
			return nil, err
		}
		sort.Strings(files)
		for _, f := range files {
			if len(only) > 0 && !only[strings.TrimSuffix(filepath.Base(f), ".yaml")] {
				continue
			}
			c, err := LoadCase(f)
			if err != nil {
				return nil, err
			}
			if c.Suite != suite {
				return nil, fmt.Errorf("%s: suite %q does not match directory %q", f, c.Suite, suite)
			}
			if prev, dup := seen[c.ID]; dup {
				return nil, fmt.Errorf("duplicate case id %s in %s and %s", c.ID, prev, f)
			}
			seen[c.ID] = f
			out = append(out, c)
		}
	}
	return out, nil
}

// LoadCase reads and validates one case file. Unknown keys are errors.
func LoadCase(path string) (*Case, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	dec := yaml.NewDecoder(strings.NewReader(string(raw)))
	dec.KnownFields(true)
	var c Case
	if err := dec.Decode(&c); err != nil {
		return nil, fmt.Errorf("%s: %w", path, err)
	}
	c.Path = path
	if c.Invoke == "" {
		c.Invoke = "natural"
	}
	if err := c.Validate(); err != nil {
		return nil, err
	}
	return &c, nil
}