Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

23.6 KB
Raw Blame
package agent_test

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"reflect"
	"slices"
	"strings"
	"sync"
	"testing"
	"testing/fstest"
	"time"

	"github.com/NandhaKishorM/laya/laya-go/agent"
	"github.com/NandhaKishorM/laya/laya-go/infer"
	"github.com/NandhaKishorM/laya/laya-go/internal/fixtures"
	"github.com/NandhaKishorM/laya/laya-go/schema"
	"github.com/NandhaKishorM/laya/laya-go/sequence"
	"github.com/NandhaKishorM/laya/laya-go/tokenizer"
)

// run is what a fake session saw of one batch: its shape and, per row, checksums that name the
// row. Row order inside a run and run order are what the chunking decides.
type run struct {
	Rows, Length, Markers int
	IDSums, Tokens        []int64
	MarkerSums, QTypes    []int64
}

// fakeSession stands in for onnxruntime. Its outputs are a function of the integers it is fed
// (exact in float32), which scripts/gen_go_fixtures.py computes the same way, so what the agent
// feeds it decides the answers.
type fakeSession struct {
	mu     sync.Mutex
	runs   []run
	pads   [][]int64 // per run, the distinct ids under a zero mask
	closes int
	// before, when set, runs at the start of each Run with the number of runs so far.
	before func(n int) error
	// shape, when set, edits the output before it is returned.
	shape func(*infer.Output)
	// onCtx, when set, sees the context each Run was given.
	onCtx func(context.Context)
}

func (s *fakeSession) Run(ctx context.Context, b infer.Batch) (infer.Output, error) {
	if s.onCtx != nil {
		s.onCtx(ctx)
	}
	if err := b.Validate(); err != nil {
		return infer.Output{}, err
	}
	r := run{Rows: b.Rows, Length: b.Length, Markers: b.Markers}
	var pads []int64
	out := infer.Output{Logits: make([]float32, b.Rows*b.Markers), ActLogits: make([]float32, b.Rows*2)}
	for i := 0; i < b.Rows; i++ {
		var sum, tokens, markers int64
		for j := 0; j < b.Length; j++ {
			if b.AttentionMask[i*b.Length+j] == 1 {
				sum += b.InputIDs[i*b.Length+j]
				tokens++
			} else if id := b.InputIDs[i*b.Length+j]; !slices.Contains(pads, id) {
				pads = append(pads, id)
			}
		}
		for j := 0; j < b.Markers; j++ {
			if b.MarkerMask[i*b.Markers+j] {
				markers += b.MarkerPos[i*b.Markers+j]
			}
			out.Logits[i*b.Markers+j] = float32((sum+31*int64(j))%97-48) / 8
		}
		for c := 0; c < 2; c++ {
			out.ActLogits[i*2+c] = float32((sum+17*int64(c))%97-48) / 8
		}
		r.IDSums = append(r.IDSums, sum)
		r.Tokens = append(r.Tokens, tokens)
		r.MarkerSums = append(r.MarkerSums, markers)
		r.QTypes = append(r.QTypes, b.QType[i])
	}
	s.mu.Lock()
	n := len(s.runs)
	s.runs = append(s.runs, r)
	s.pads = append(s.pads, pads)
	before, shape := s.before, s.shape
	s.mu.Unlock()
	if before != nil {
		if err := before(n); err != nil {
			return infer.Output{}, err
		}
	}
	if shape != nil {
		shape(&out)
	}
	return out, nil
}

func (s *fakeSession) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.closes++
	return nil
}

func (s *fakeSession) seen() []run {
	s.mu.Lock()
	defer s.mu.Unlock()
	return append([]run(nil), s.runs...)
}

// toyCheckpoint is a checkpoint directory holding the toy tokenizer and the given config.
func toyCheckpoint(config string) fstest.MapFS {
	spec, cfg := fixtures.ToyTokenizer()
	return fstest.MapFS{
		"rl_agent_config.json":            {Data: []byte(config)},
		"tokenizer/tokenizer.json":        {Data: spec},
		"tokenizer/tokenizer_config.json": {Data: cfg},
	}
}

func open(t *testing.T, config string) (*agent.Agent, *fakeSession) {
	t.Helper()
	s := &fakeSession{}
	a, err := agent.Open(toyCheckpoint(config), s)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { a.Close() })
	return a, s
}

func questions(t *testing.T) schema.Questions {
	t.Helper()
	qs, err := schema.NewQuestions(
		schema.Ask("intent", schema.MustQuestion(schema.ChoiceOf("What does the customer want to do?", "refund", "status", "other"))),
		schema.Ask("urgency", schema.MustQuestion(schema.Score("How urgent?", "low", "medium", "high"))),
		schema.Ask("angry", schema.MustQuestion(schema.Noul("Angry?"))),
	)
	if err != nil {
		t.Fatal(err)
	}
	return qs
}

func jsonOf(t *testing.T, v any) string {
	t.Helper()
	b, err := json.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func TestLoadConfig(t *testing.T) {
	cases := []struct {
		name     string
		config   string
		want     agent.Config
		warnings []string // substrings of the one warning, if any
	}{
		{"defaults", `{}`, agent.Config{MaxLen: 512, HeadMaxLen: 192, OptionLayout: "sequential"}, nil},
		{"multilingual", `{"max_len": 1024, "head_max_len": 256, "temperature": [1.0, 1.0, 1.0]}`,
			agent.Config{MaxLen: 1024, HeadMaxLen: 256, OptionLayout: "sequential"}, nil},
		{"explicit layout", `{"option_layout": "sequential", "max_len": 64}`,
			agent.Config{MaxLen: 64, HeadMaxLen: 192, OptionLayout: "sequential"}, nil},
		{"unknown keys are ignored", `{"head_layers": 2, "training": {"hours": 1.96}}`,
			agent.Config{MaxLen: 512, HeadMaxLen: 192, OptionLayout: "sequential"}, nil},
		{"in-range temperatures are quiet", `{"temperature": [1.6, 1.25, 2], "temperature_by_options": {"choice:2": 1.9}}`,
			agent.Config{MaxLen: 512, HeadMaxLen: 192, OptionLayout: "sequential"}, nil},
		{"clamped temperatures warn once", `{"temperature": [1.0, 9, 0.1], "temperature_by_options": {"choice:11+": 0.1006}}`,
			agent.Config{MaxLen: 512, HeadMaxLen: 192, OptionLayout: "sequential"},
			[]string{"[0.5, 5]", "choice:11+=0.1006 -> 0.5", "temperature[1]=9 -> 5", "temperature[2]=0.1 -> 0.5"}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got, err := agent.LoadConfig(toyCheckpoint(c.config))
			if err != nil {
				t.Fatal(err)
			}
			warnings := got.Warnings
			got.Warnings = nil
			if got.MaxLen != c.want.MaxLen || got.HeadMaxLen != c.want.HeadMaxLen || got.OptionLayout != c.want.OptionLayout {
				t.Fatalf("Config = %+v, want %+v", got, c.want)
			}
			if len(c.warnings) == 0 && len(warnings) != 0 {
				t.Fatalf("warnings = %q, want none", warnings)
			}
			if len(c.warnings) > 0 {
				if len(warnings) != 1 {
					t.Fatalf("%d warnings, want one: %q", len(warnings), warnings)
				}
				for _, part := range c.warnings {
					if !strings.Contains(warnings[0], part) {
						t.Errorf("warning %q lacks %q", warnings[0], part)
					}
				}
			}
		})
	}
}

func TestLoadConfigErrors(t *testing.T) {
	cases := []struct {
		name   string
		config string // "-" for no file
		field  string
		is     error
	}{
		{"missing file", "-", "", fs.ErrNotExist},
		{"not JSON", `{`, "", nil},
		{"not an object", `[1]`, "", nil},
		{"other layout", `{"option_layout": "dual"}`, "option_layout", agent.ErrUnsupportedLayout},
		{"null layout", `{"option_layout": null}`, "option_layout", agent.ErrUnsupportedLayout},
		{"max_len is text", `{"max_len": "512"}`, "max_len", nil},
		{"max_len is a float", `{"max_len": 512.5}`, "max_len", nil},
		{"max_len is zero", `{"max_len": 0}`, "max_len", nil},
		{"head_max_len is negative", `{"head_max_len": -4}`, "head_max_len", nil},
		{"temperature is not a list", `{"temperature": 1.5}`, "temperature", nil},
		{"temperature has two values", `{"temperature": [1, 1]}`, "temperature", nil},
		{"buckets are a list", `{"temperature_by_options": [1]}`, "temperature_by_options", nil},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			fsys := toyCheckpoint(c.config)
			if c.config == "-" {
				delete(fsys, "rl_agent_config.json")
			}
			_, err := agent.LoadConfig(fsys)
			var ce *agent.ConfigError
			if !errors.As(err, &ce) || ce.Field != c.field {
				t.Fatalf("LoadConfig = %v, want a ConfigError for %q", err, c.field)
			}
			if c.is != nil && !errors.Is(err, c.is) {
				t.Fatalf("LoadConfig = %v, want it to wrap %v", err, c.is)
			}
			if !strings.Contains(err.Error(), "rl_agent_config.json") {
				t.Errorf("error %q does not name the file", err)
			}
		})
	}
}

// TestConfigInUse pins that the checkpoint's own max_len is the budget, not the default.
func TestConfigInUse(t *testing.T) {
	long := strings.Repeat("x", 800)
	one := func(config string) (agent.Config, schema.Usage) {
		a, _ := open(t, config)
		p, err := a.Predict(context.Background(), long, questions(t))
		if err != nil {
			t.Fatal(err)
		}
		return a.Config(), p.Usage
	}
	cfg, u := one(`{"max_len": 1024, "head_max_len": 256}`)
	if cfg.MaxLen != 1024 || cfg.HeadMaxLen != 256 {
		t.Fatalf("Config = %+v", cfg)
	}
	if u.Truncated || u.StateTokens != 800 {
		t.Errorf("max_len 1024: usage = %s", jsonOf(t, u))
	}
	if _, u = one(`{}`); !u.Truncated || u.StateTokensDropped < 280 {
		t.Errorf("default max_len 512: usage = %s", jsonOf(t, u))
	}
}

func TestConfigReturnsACopy(t *testing.T) {
	a, _ := open(t, `{"temperature": [9, 1, 1]}`)
	a.Config().Warnings[0] = "changed"
	if w := a.Config().Warnings[0]; w == "changed" {
		t.Fatal("Config shares its Warnings with the agent")
	}
}

func TestOpenClosesTheSessionOnError(t *testing.T) {
	noTokenizer := toyCheckpoint(`{}`)
	delete(noTokenizer, "tokenizer/tokenizer.json")
	for name, fsys := range map[string]fs.FS{
		"bad config":    toyCheckpoint(`{"option_layout": "x"}`),
		"no tokenizer":  noTokenizer,
		"no config":     fstest.MapFS{},
		"empty tokeniz": fstest.MapFS{"rl_agent_config.json": {Data: []byte(`{}`)}, "tokenizer.json": {Data: []byte(`{}`)}},
	} {
		t.Run(name, func(t *testing.T) {
			s := &fakeSession{}
			if a, err := agent.Open(fsys, s); err == nil || a != nil {
				t.Fatalf("Open = %v, %v", a, err)
			}
			if s.closes != 1 {
				t.Fatalf("the session was closed %d times, want once", s.closes)
			}
		})
	}
	if _, err := agent.Open(toyCheckpoint(`{}`), nil); err == nil {
		t.Fatal("Open accepted a nil session")
	}
}

func TestOpenWithBadLayoutWraps(t *testing.T) {
	_, err := agent.Open(toyCheckpoint(`{"option_layout": "dual"}`), &fakeSession{})
	if !errors.Is(err, agent.ErrUnsupportedLayout) || !strings.Contains(err.Error(), "'dual'") {
		t.Fatalf("Open = %v", err)
	}
}

func TestClose(t *testing.T) {
	s := &fakeSession{}
	a, err := agent.Open(toyCheckpoint(`{}`), s)
	if err != nil {
		t.Fatal(err)
	}
	if err := a.Close(); err != nil {
		t.Fatal(err)
	}
	if err := a.Close(); err != nil {
		t.Fatalf("second Close = %v", err)
	}
	if s.closes != 1 {
		t.Fatalf("session closed %d times, want once", s.closes)
	}
	if _, err := a.Predict(context.Background(), "x", questions(t)); !errors.Is(err, agent.ErrClosed) {
		t.Fatalf("Predict after Close = %v", err)
	}
	if _, err := a.PredictBatch(context.Background(), []any{"x"}, questions(t)); !errors.Is(err, agent.ErrClosed) {
		t.Fatalf("PredictBatch after Close = %v", err)
	}
	if len(s.seen()) != 0 {
		t.Fatal("the closed agent still ran the session")
	}
}

func TestEmptyQuestionsAndStates(t *testing.T) {
	a, s := open(t, `{}`)
	ctx := context.Background()
	preds, err := a.PredictBatch(ctx, []any{"one", "two"}, schema.Questions{})
	if err != nil || len(preds) != 2 {
		t.Fatalf("PredictBatch = %v, %v", preds, err)
	}
	want := `{"model":"laya-rl-agent-onnx","answers":{},"usage":{"input_tokens":0,"output_tokens":0,"state_tokens":0,"state_tokens_dropped":0,"truncated":false,"truncated_questions":[]}}`
	for _, p := range preds {
		if got := jsonOf(t, p); got != want {
			t.Errorf("got %s\nwant %s", got, want)
		}
	}
	for _, states := range [][]any{nil, {}} {
		preds, err := a.PredictBatch(ctx, states, questions(t))
		if err != nil || preds == nil || len(preds) != 0 {
			t.Errorf("PredictBatch(%v) = %#v, %v, want an empty non-nil slice", states, preds, err)
		}
	}
	if len(s.seen()) != 0 {
		t.Fatal("the session ran for nothing")
	}
}

func TestStateErrors(t *testing.T) {
	a, s := open(t, `{}`)
	ctx := context.Background()
	if _, err := a.PredictBatch(ctx, []any{"fine", nil}, questions(t)); err == nil || !strings.Contains(err.Error(), "state 1: is nil") {
		t.Errorf("nil state: %v", err)
	}
	if _, err := a.Predict(ctx, struct{ X int }{1}, questions(t)); !errors.Is(err, sequence.ErrUnsupportedState) {
		t.Errorf("struct state: %v", err)
	}
	if len(s.seen()) != 0 {
		t.Error("a refused request still ran the session")
	}
}

func TestMarkersErrorPassesThrough(t *testing.T) {
	a, _ := open(t, `{"max_len": 40, "head_max_len": 30}`)
	labels := make([]string, 12)
	for i := range labels {
		labels[i] = fmt.Sprintf("option number %d", i)
	}
	qs, err := schema.NewQuestions(
		schema.Ask("fine", schema.MustQuestion(schema.Noul("ok"))),
		schema.Ask("wide", schema.MustQuestion(schema.ChoiceOf("pick", labels...))))
	if err != nil {
		t.Fatal(err)
	}
	_, err = a.Predict(context.Background(), "state", qs)
	var me *sequence.MarkersError
	if !errors.As(err, &me) || me.ID != "wide" || me.Total != 12 || me.MaxLen != 40 {
		t.Fatalf("Predict = %v (%+v)", err, me)
	}
}

// TestOneOptionQuestion pins that a question with one option still sends two marker slots.
func TestOneOptionQuestion(t *testing.T) {
	a, s := open(t, `{}`)
	qs, err := schema.NewQuestions(schema.Ask("only", schema.MustQuestion(schema.ChoiceOf("pick", "yes"))))
	if err != nil {
		t.Fatal(err)
	}
	p, err := a.Predict(context.Background(), "state", qs)
	if err != nil {
		t.Fatal(err)
	}
	if runs := s.seen(); len(runs) != 1 || runs[0].Markers != 2 {
		t.Fatalf("runs = %+v, want one run with 2 marker slots", runs)
	}
	if got := jsonOf(t, p.Answers); !strings.Contains(got, `"choice":"yes"`) || !strings.Contains(got, `"yes":1`) {
		t.Fatalf("answers = %s", got)
	}
}

func TestOutputShapeMismatch(t *testing.T) {
	a, s := open(t, `{}`)
	s.shape = func(o *infer.Output) { o.Logits = o.Logits[1:] }
	var se *infer.ShapeError
	if _, err := a.Predict(context.Background(), "state", questions(t)); !errors.As(err, &se) || se.Name != "logits" {
		t.Fatalf("Predict = %v", err)
	}
	s.shape = func(o *infer.Output) { o.ActLogits = nil }
	if _, err := a.Predict(context.Background(), "state", questions(t)); !errors.As(err, &se) || se.Name != "act_logits" {
		t.Fatalf("Predict = %v", err)
	}
}

func TestSessionErrorIsWrapped(t *testing.T) {
	a, s := open(t, `{}`)
	boom := errors.New("boom")
	s.before = func(int) error { return boom }
	if _, err := a.Predict(context.Background(), "state", questions(t)); !errors.Is(err, boom) {
		t.Fatalf("Predict = %v", err)
	}
}

func TestNonFiniteLogitsAreRefused(t *testing.T) {
	a, s := open(t, `{}`)
	s.shape = func(o *infer.Output) { o.Logits[0] = float32(nan()) }
	if _, err := a.Predict(context.Background(), "state", questions(t)); err == nil || !strings.Contains(err.Error(), `question "intent"`) && !strings.Contains(err.Error(), "question intent") {
		t.Fatalf("Predict = %v", err)
	}
}

func nan() float64 { z := 0.0; return z / z }

func varied() []any {
	var states []any
	for _, n := range []int{30, 5, 20, 10, 25, 1, 17, 8, 22, 3, 14} {
		states = append(states, strings.Repeat("a", n)+fmt.Sprint(n))
	}
	states = append(states, []any{"hello", "there"}, map[string]any{"k": "v"}, "")
	return states
}

// TestResultsAreTheSameWhateverTheBatching is the contract that makes chunking and sorting safe:
// each state's prediction depends on its own rows only, and comes back at its own index.
func TestResultsAreTheSameWhateverTheBatching(t *testing.T) {
	ctx := context.Background()
	states := varied()
	qs := questions(t)
	a, _ := open(t, `{}`)
	var want []string
	for _, st := range states {
		p, err := a.Predict(ctx, st, qs)
		if err != nil {
			t.Fatal(err)
		}
		want = append(want, jsonOf(t, p.Answers))
	}
	if want[0] == want[1] {
		t.Fatal("the states are not told apart by the fake session")
	}
	for _, size := range []int{0, 1, 2, 3, 5, len(states) - 1, len(states), len(states) + 4} {
		for _, sorted := range []bool{false, true} {
			opts := []agent.CallOption{agent.WithBatchSize(size)}
			if sorted {
				opts = append(opts, agent.WithSortByLength(true))
			}
			got, err := a.PredictBatch(ctx, states, qs, opts...)
			if err != nil || len(got) != len(states) {
				t.Fatalf("batch %d sorted %v: %v (%d results)", size, sorted, err, len(got))
			}
			for i, p := range got {
				if jsonOf(t, p.Answers) != want[i] {
					t.Errorf("batch %d sorted %v: state %d got another state's answers", size, sorted, i)
				}
			}
		}
	}
}

func lengths(runs []run) []int {
	var out []int
	for _, r := range runs {
		out = append(out, r.Length)
	}
	return out
}

func TestChunkingAndSorting(t *testing.T) {
	ctx := context.Background()
	states := varied()[:11] // distinct lengths, none a list
	qs := questions(t)
	record := func(opts ...agent.CallOption) []run {
		a, s := open(t, `{}`)
		if _, err := a.PredictBatch(ctx, states, qs, opts...); err != nil {
			t.Fatal(err)
		}
		return s.seen()
	}
	rowsOf := func(runs []run) (rows []int) {
		for _, r := range runs {
			rows = append(rows, r.Rows)
		}
		return rows
	}
	if got := rowsOf(record()); !reflect.DeepEqual(got, []int{33}) {
		t.Errorf("no batch size: runs of %v rows", got)
	}
	if got := rowsOf(record(agent.WithBatchSize(4))); !reflect.DeepEqual(got, []int{12, 12, 9}) {
		t.Errorf("batch 4: runs of %v rows, want 4 states x 3 questions each", got)
	}
	// Sorting reorders only when 1 < chunk < states: otherwise the runs are the unsorted ones.
	for _, size := range []int{1, 11, 40} {
		plain := record(agent.WithBatchSize(size))
		sorted := record(agent.WithBatchSize(size), agent.WithSortByLength(true))
		if !reflect.DeepEqual(plain, sorted) {
			t.Errorf("batch %d: sorting changed the runs", size)
		}
	}
	unsorted := lengths(record(agent.WithBatchSize(3)))
	sorted := lengths(record(agent.WithBatchSize(3), agent.WithSortByLength(true)))
	for i := 1; i < len(sorted); i++ {
		if sorted[i] < sorted[i-1] {
			t.Errorf("sorted run lengths %v are not ascending", sorted)
		}
	}
	if reflect.DeepEqual(unsorted, sorted) {
		t.Errorf("sorting changed nothing: %v", sorted)
	}
}

// TestUsageIsPerState pins that a state's input_tokens are the tokens of its own rows only.
func TestUsageIsPerState(t *testing.T) {
	a, s := open(t, `{}`)
	states := []any{strings.Repeat("a", 9), strings.Repeat("b", 31), "c"}
	got, err := a.PredictBatch(context.Background(), states, questions(t))
	if err != nil {
		t.Fatal(err)
	}
	runs := s.seen()
	if len(runs) != 1 {
		t.Fatalf("%d runs", len(runs))
	}
	var total int64
	for i, p := range got {
		var own int64
		for _, n := range runs[0].Tokens[i*3 : i*3+3] {
			own += n
		}
		total += own
		if int64(p.Usage.InputTokens) != own {
			t.Errorf("state %d: input_tokens %d, want %d", i, p.Usage.InputTokens, own)
		}
		if p.Usage.OutputTokens != 0 {
			t.Errorf("state %d: output_tokens %d", i, p.Usage.OutputTokens)
		}
	}
	if got[0].Usage.StateTokens != 9 || got[1].Usage.StateTokens != 31 || got[2].Usage.StateTokens != 1 {
		t.Errorf("state_tokens = %d, %d, %d", got[0].Usage.StateTokens, got[1].Usage.StateTokens, got[2].Usage.StateTokens)
	}
	if got[0].Usage.InputTokens == got[1].Usage.InputTokens || int64(got[0].Usage.InputTokens) == total {
		t.Error("input_tokens do not differ per state")
	}
}

// TestTruncationReport pins state_tokens_dropped as the worst question, and truncated_questions as
// the ones that were cut, when the questions leave different room.
func TestTruncationReport(t *testing.T) {
	a, _ := open(t, `{"max_len": 200, "head_max_len": 120}`)
	wide := make([]string, 10)
	for i := range wide {
		wide[i] = fmt.Sprintf("a fairly long option text number %d", i)
	}
	qs, err := schema.NewQuestions(
		schema.Ask("small", schema.MustQuestion(schema.Noul("ok"))),
		schema.Ask("wide", schema.MustQuestion(schema.ChoiceOf("pick one", wide...))),
		schema.Ask("small2", schema.MustQuestion(schema.Noul("ok"))))
	if err != nil {
		t.Fatal(err)
	}
	p, err := a.Predict(context.Background(), strings.Repeat("x", 100), qs)
	if err != nil {
		t.Fatal(err)
	}
	u := p.Usage
	spec, cfg := fixtures.ToyTokenizer()
	tok, err := tokenizer.Parse(spec, cfg)
	if err != nil {
		t.Fatal(err)
	}
	worst, cut := 0, []string{}
	for i := 0; i < qs.Len(); i++ {
		room, err := sequence.StateRoom(tok, qs.At(i).Question, sequence.Budget{MaxLen: 200, HeadMaxLen: 120})
		if err != nil {
			t.Fatal(err)
		}
		if dropped := 100 - room; dropped > 0 {
			worst = max(worst, dropped)
			cut = append(cut, qs.At(i).ID)
		}
	}
	if worst == 0 || len(cut) != 1 || cut[0] != "wide" {
		t.Fatalf("the test no longer cuts only the wide question: %v, %d", cut, worst)
	}
	if !u.Truncated || u.StateTokensDropped != worst || !reflect.DeepEqual(u.TruncatedQuestions, cut) || u.StateTokens != 100 {
		t.Fatalf("usage = %s, want dropped %d for %v", jsonOf(t, u), worst, cut)
	}
}

func TestLanguageWithoutOverridesChangesNothing(t *testing.T) {
	a, _ := open(t, `{}`)
	plain, err := a.Predict(context.Background(), "state", questions(t))
	if err != nil {
		t.Fatal(err)
	}
	tagged, err := a.Predict(context.Background(), "state", questions(t), agent.WithLanguage("yo-NG"))
	if err != nil || jsonOf(t, plain) != jsonOf(t, tagged) {
		t.Fatalf("a language tag changed the answer: %v", err)
	}
}

func TestContextStopsTheBatchBetweenChunks(t *testing.T) {
	states := varied()
	qs := questions(t)
	t.Run("cancelled during the first chunk", func(t *testing.T) {
		a, s := open(t, `{}`)
		ctx, cancel := context.WithCancel(context.Background())
		s.before = func(n int) error {
			if n == 0 {
				cancel()
			}
			return nil
		}
		got, err := a.PredictBatch(ctx, states, qs, agent.WithBatchSize(3))
		if err != ctx.Err() || got != nil {
			t.Fatalf("PredictBatch = %v, %v, want ctx.Err()", got, err)
		}
		if n := len(s.seen()); n != 1 {
			t.Fatalf("%d runs after the cancel, want 1", n)
		}
	})
	t.Run("cancelled before the call", func(t *testing.T) {
		a, s := open(t, `{}`)
		ctx, cancel := context.WithCancel(context.Background())
		cancel()
		if _, err := a.PredictBatch(ctx, states, qs, agent.WithBatchSize(3)); !errors.Is(err, context.Canceled) {
			t.Fatalf("PredictBatch = %v", err)
		}
		if len(s.seen()) != 0 {
			t.Fatal("ran with a cancelled context")
		}
	})
	t.Run("deadline between sorted chunks", func(t *testing.T) {
		a, s := open(t, `{}`)
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
		defer cancel()
		s.before = func(int) error { <-ctx.Done(); return nil }
		if _, err := a.PredictBatch(ctx, states, qs, agent.WithBatchSize(2), agent.WithSortByLength(true)); !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("PredictBatch = %v", err)
		}
		if n := len(s.seen()); n != 1 {
			t.Fatalf("%d runs, want 1", n)
		}
	})
}

func TestConcurrentPredict(t *testing.T) {
	a, _ := open(t, `{}`)
	qs := questions(t)
	states := varied()
	want := make([]string, len(states))
	for i, st := range states {
		p, err := a.Predict(context.Background(), st, qs)
		if err != nil {
			t.Fatal(err)
		}
		want[i] = jsonOf(t, p)
	}
	var wg sync.WaitGroup
	errs := make(chan error, 8)
	for g := 0; g < 8; g++ {
		wg.Add(1)
		go func(g int) {
			defer wg.Done()
			for round := 0; round < 5; round++ {
				if g%2 == 0 {
					i := (g + round) % len(states)
					p, err := a.Predict(context.Background(), states[i], qs)
					if err != nil || jsonOf(t, p) != want[i] {
						errs <- fmt.Errorf("goroutine %d: Predict(%d) = %v", g, i, err)
						return
					}
					continue
				}
				got, err := a.PredictBatch(context.Background(), states, qs, agent.WithBatchSize(3+g), agent.WithSortByLength(true))
				if err != nil {
					errs <- err
					return
				}
				for i, p := range got {
					if jsonOf(t, p) != want[i] {
						errs <- fmt.Errorf("goroutine %d: state %d differs", g, i)
						return
					}
				}
			}
		}(g)
	}
	wg.Wait()
	close(errs)
	for err := range errs {
		t.Error(err)
	}
}

func TestCloseWaitsForCallsInFlight(t *testing.T) {
	s := &fakeSession{}
	a, err := agent.Open(toyCheckpoint(`{}`), s)
	if err != nil {
		t.Fatal(err)
	}
	started, release := make(chan struct{}), make(chan struct{})
	s.before = func(int) error { close(started); <-release; return nil }
	done := make(chan error, 1)
	go func() { _, err := a.Predict(context.Background(), "state", questions(t)); done <- err }()
	<-started
	closed := make(chan struct{})
	go func() { a.Close(); close(closed) }()
	select {
	case <-closed:
		t.Fatal("Close returned while a call was still running")
	case <-time.After(50 * time.Millisecond):
	}
	close(release)
	if err := <-done; err != nil {
		t.Fatal(err)
	}
	<-closed
}