Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

13.9 KB
Raw Blame
//go:build cgo

package onnx

import (
	"context"
	"encoding/json"
	"errors"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"sync"
	"testing"

	"github.com/NandhaKishorM/laya/laya-go/agent"
	"github.com/NandhaKishorM/laya/laya-go/internal/fixtures"
	"github.com/NandhaKishorM/laya/laya-go/schema"
	"github.com/NandhaKishorM/laya/laya-go/sequence"
)

// Comparison rules, from laya-java's PredictionGolden: structure and usage exact, numbers within
// a tolerance because Python computes them on whatever CPU it ran on.
const (
	tolerance      = 2e-4 // probabilities, confidences, act_probability, noul
	scoreTolerance = 8e-4 // score, the expectation over the probabilities
	decisiveGap    = 1e-3 // the chosen label is compared when the top two differ by more
	maxDiffering   = 0.05 // of all compared numbers may differ at all
)

type tally struct {
	t                  testing.TB
	where              string
	numbers, differing int
	worst              float64
	labels, ties       int
	labelDiffs         int
	failures           int
}

func (c *tally) failf(format string, args ...any) {
	c.failures++
	if c.failures <= 10 {
		c.t.Errorf(c.where+": "+format, args...)
	}
}

func (c *tally) number(path string, got, want json.Number, tol float64) {
	g, errG := got.Float64()
	w, errW := want.Float64()
	if errG != nil || errW != nil {
		c.failf("%s: %q vs %q", path, got, want)
		return
	}
	c.numbers++
	if got != want {
		c.differing++
	}
	d := math.Abs(g - w)
	c.worst = math.Max(c.worst, d)
	if d > tol {
		c.failf("%s: got %v want %v (off by %.2e, tolerance %.0e)", path, g, w, d, tol)
	}
}

// value compares got to want. Strings, booleans, keys and their order are exact.
func (c *tally) value(path, key string, got, want any) {
	switch w := want.(type) {
	case fixtures.Object:
		g, ok := got.(fixtures.Object)
		if !ok || !reflect.DeepEqual(g.Keys(), w.Keys()) {
			c.failf("%s: keys %v, want %v", path, got, w.Keys())
			return
		}
		for _, m := range w {
			v, _ := g.Get(m.Key)
			c.value(path+"."+m.Key, m.Key, v, m.Value)
		}
	case []any:
		g, ok := got.([]any)
		if !ok || len(g) != len(w) {
			c.failf("%s: %v, want %d items", path, got, len(w))
			return
		}
		for i := range w {
			c.value(path+"[]", key, g[i], w[i])
		}
	case json.Number:
		g, ok := got.(json.Number)
		if !ok {
			c.failf("%s: %v, want a number", path, got)
			return
		}
		tol := tolerance
		if key == "score" {
			tol = scoreTolerance
		}
		c.number(path, g, w, tol)
	default:
		if !reflect.DeepEqual(got, want) {
			c.failf("%s: %v, want %v", path, got, want)
		}
	}
}

func without(o fixtures.Object, key string) fixtures.Object {
	var out fixtures.Object
	for _, m := range o {
		if m.Key != key {
			out = append(out, m)
		}
	}
	return out
}

// answer compares one answer; the chosen label is held to Python's only when it was decisive.
func (c *tally) answer(path string, got, want fixtures.Object) {
	if typ, _ := want.Get("type"); typ == "choice" {
		var ps []float64
		for _, m := range fixtures.Obj(c.t, want.Must("probabilities")) {
			ps = append(ps, fixtures.Float(c.t, m.Value))
		}
		sort.Sort(sort.Reverse(sort.Float64Slice(ps)))
		if len(ps) > 1 && ps[0]-ps[1] <= decisiveGap {
			c.ties++
			c.value(path, "", without(got, "choice"), without(want, "choice"))
			return
		}
		c.labels++
		if got.Must("choice") != want.Must("choice") {
			c.labelDiffs++
		}
	}
	c.value(path, "", got, want)
}

// prediction compares one result: model and usage exactly, answers by the rules above.
func (c *tally) prediction(path string, p schema.Prediction, want fixtures.Object) {
	data, err := json.Marshal(p)
	if err != nil {
		c.t.Fatal(err)
	}
	parsed, err := fixtures.Parse(data)
	if err != nil {
		c.t.Fatal(err)
	}
	got := fixtures.Obj(c.t, parsed)
	c.value(path+".model", "model", got.Must("model"), want.Must("model"))
	if g, w := fixtures.Marshal(c.t, got.Must("usage")), fixtures.Marshal(c.t, want.Must("usage")); string(g) != string(w) {
		c.failf("%s usage\n got %s\nwant %s", path, g, w)
	}
	gotAnswers, wantAnswers := fixtures.Obj(c.t, got.Must("answers")), fixtures.Obj(c.t, want.Must("answers"))
	if !reflect.DeepEqual(gotAnswers.Keys(), wantAnswers.Keys()) {
		c.failf("%s answer ids %v, want %v", path, gotAnswers.Keys(), wantAnswers.Keys())
		return
	}
	for _, m := range wantAnswers {
		g, _ := gotAnswers.Get(m.Key)
		c.answer(path+"."+m.Key, fixtures.Obj(c.t, g), fixtures.Obj(c.t, m.Value))
	}
}

func (c *tally) report(name string) {
	c.t.Logf("%s: %d numbers compared, %d differ at all, worst deviation %.2e, %d labels checked (%d differ), %d near-ties skipped",
		name, c.numbers, c.differing, c.worst, c.labels, c.labelDiffs, c.ties)
}

func (c *tally) finish(name string, minNumbers int) {
	c.report(name)
	if c.numbers < minNumbers {
		c.t.Errorf("%s: only %d numbers were compared", name, c.numbers)
	}
	if float64(c.differing) > maxDiffering*float64(c.numbers) {
		c.t.Errorf("%s: %d of %d numbers differ (more than %.0f%%)", name, c.differing, c.numbers, maxDiffering*100)
	}
}

func openAgent(t testing.TB, p paths, s *Session) *agent.Agent {
	t.Helper()
	a, err := agent.Open(os.DirFS(filepath.Join(p.checkpoints, "multilingual")), keepOpen{s})
	if err != nil {
		t.Fatalf("agent.Open: %v", err)
	}
	t.Cleanup(func() { a.Close() })
	return a
}

type singleCase struct {
	name  string
	state any
	lang  string
	qs    schema.Questions
	want  fixtures.Object
}

func loadState(t testing.TB, v any) any {
	t.Helper()
	st, err := sequence.ParseState(fixtures.Marshal(t, v))
	if err != nil {
		t.Fatal(err)
	}
	return st
}

func loadQuestions(t testing.TB, v any) schema.Questions {
	t.Helper()
	qs, err := schema.ParseQuestions(fixtures.Marshal(t, v))
	if err != nil {
		t.Fatal(err)
	}
	return qs
}

func singles(t testing.TB) []singleCase {
	t.Helper()
	golden := fixtures.Load(t, "predict.json")
	var out []singleCase
	for _, m := range fixtures.Obj(t, golden.Must("single")) {
		rec := fixtures.Obj(t, m.Value)
		out = append(out, singleCase{
			name: m.Key, state: loadState(t, rec.Must("state")), lang: fixtures.StrOrEmpty(t, rec.Must("lang")),
			qs: loadQuestions(t, rec.Must("questions")), want: rec,
		})
	}
	return out
}

func (sc singleCase) options() []agent.CallOption {
	if sc.lang == "" {
		return nil
	}
	return []agent.CallOption{agent.WithLanguage(sc.lang)}
}

// TestPredictGolden holds the whole path (agent, collator, this session, ONNX Runtime) to the
// answers Python's ONNXAgent recorded on the same graph.
func TestPredictGolden(t *testing.T) {
	p := need(t)
	a := openAgent(t, p, sharedSession(t))
	golden := fixtures.Load(t, "predict.json")
	if cp := fixtures.Str(t, golden.Must("checkpoint")); cp != "multilingual" {
		t.Fatalf("the golden was recorded on the %s checkpoint", cp)
	}
	ctx := context.Background()

	total := &tally{t: t}
	t.Run("single", func(t *testing.T) {
		cases := singles(t)
		if len(cases) != 6 {
			t.Fatalf("%d single cases", len(cases))
		}
		c := &tally{t: t}
		for _, sc := range cases {
			c.where = sc.name
			pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)
			if err != nil {
				t.Fatalf("%s: %v", sc.name, err)
			}
			c.prediction(sc.name, pred, sc.want)
		}
		c.finish("single", 60)
		merge(total, c)
	})

	t.Run("batch", func(t *testing.T) {
		batches := fixtures.List(t, golden.Must("batch"))
		if len(batches) != 3 {
			t.Fatalf("%d batch configurations", len(batches))
		}
		for _, raw := range batches {
			rec := fixtures.Obj(t, raw)
			var opts []agent.CallOption
			name := "all-in-one"
			if n := rec.Must("batch_size"); n != nil {
				size := fixtures.Int(t, n)
				opts = append(opts, agent.WithBatchSize(size))
				name = "batch-" + string(rune('0'+size))
			}
			if fixtures.Bool(t, rec.Must("sort_by_length")) {
				opts = append(opts, agent.WithSortByLength(true))
				name += "-sorted"
			}
			t.Run(name, func(t *testing.T) {
				var states []any
				for _, s := range fixtures.List(t, rec.Must("states")) {
					states = append(states, loadState(t, s))
				}
				preds, err := a.PredictBatch(ctx, states, loadQuestions(t, rec.Must("questions")), opts...)
				if err != nil {
					t.Fatal(err)
				}
				want := fixtures.List(t, rec.Must("results"))
				if len(preds) != len(want) || len(want) != 9 {
					t.Fatalf("%d results, golden has %d", len(preds), len(want))
				}
				c := &tally{t: t}
				for i, pred := range preds {
					c.where = name
					c.prediction(name+"/state"+string(rune('0'+i)), pred, fixtures.Obj(t, want[i]))
				}
				c.finish(name, 80)
				merge(total, c)
			})
		}
	})
	total.report("all")
	if total.labelDiffs != 0 {
		t.Errorf("%d decisive labels differ from Python", total.labelDiffs)
	}
}

func merge(dst, src *tally) {
	dst.numbers += src.numbers
	dst.differing += src.differing
	dst.worst = math.Max(dst.worst, src.worst)
	dst.labels += src.labels
	dst.labelDiffs += src.labelDiffs
	dst.ties += src.ties
}

// TestConcurrentPredict calls one agent from eight goroutines at once; each answer must equal
// the serial one, and -race watches the Go side.
func TestConcurrentPredict(t *testing.T) {
	p := need(t)
	a := openAgent(t, p, sharedSession(t))
	cases := singles(t)
	ctx := context.Background()
	serial := make([][]byte, len(cases))
	for i, sc := range cases {
		pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)
		if err != nil {
			t.Fatal(err)
		}
		serial[i], _ = json.Marshal(pred)
	}
	var wg sync.WaitGroup
	errs := make(chan error, 8*len(cases))
	for g := 0; g < 8; g++ {
		wg.Add(1)
		go func(g int) {
			defer wg.Done()
			for k := range cases {
				i := (k + g) % len(cases)
				pred, err := a.Predict(ctx, cases[i].state, cases[i].qs, cases[i].options()...)
				if err != nil {
					errs <- err
					continue
				}
				if got, _ := json.Marshal(pred); string(got) != string(serial[i]) {
					errs <- errors.New(cases[i].name + ": concurrent answer differs from the serial one:\n" + string(got) + "\n" + string(serial[i]))
				}
			}
		}(g)
	}
	wg.Wait()
	close(errs)
	for err := range errs {
		t.Error(err)
	}
}

func TestPredictWithCancelledContext(t *testing.T) {
	p := need(t)
	s := sharedSession(t)
	a := openAgent(t, p, s)
	sc := singles(t)[0]
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	before := s.runs.Load()
	if _, err := a.Predict(ctx, sc.state, sc.qs); !errors.Is(err, context.Canceled) {
		t.Fatalf("Predict = %v, want context.Canceled", err)
	}
	if s.runs.Load() != before {
		t.Fatal("the model ran for a cancelled context")
	}
}

func TestRuntimeVersion(t *testing.T) {
	s := sharedSession(t)
	t.Logf("ONNX Runtime %s", s.RuntimeVersion())
	if s.RuntimeVersion() == "" {
		t.Fatal("empty RuntimeVersion")
	}
}

// TestMarkerMaskReachesTheGraph pins that marker_mask is sent as given: padded slots must come
// back different from real ones, so a session that sent all true would be caught.
func TestMarkerMaskReachesTheGraph(t *testing.T) {
	s := sharedSession(t)
	masked := makeBatch(2, 16, 4)
	for r := 0; r < masked.Rows; r++ {
		masked.MarkerMask[r*4+2], masked.MarkerMask[r*4+3] = false, false
	}
	open := makeBatch(2, 16, 4)
	a, err := s.Run(context.Background(), masked)
	if err != nil {
		t.Fatal(err)
	}
	b, err := s.Run(context.Background(), open)
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("masked %v\nopen   %v", a.Logits, b.Logits)
	same := true
	for i := range a.Logits {
		same = same && a.Logits[i] == b.Logits[i]
	}
	if same {
		t.Fatal("marker_mask has no effect on the logits")
	}
	if !masked.MarkerMask[0] || masked.MarkerMask[2] {
		t.Fatal("Run modified the batch")
	}
}

// TestTemperaturesReachTheAnswers pins that the checkpoint's temperatures are applied by type and
// by option bucket. The shipped multilingual checkpoint uses 1.0 everywhere, which hides a
// decode that ignores them, so this runs the same session under edited copies of its config.
func TestTemperaturesReachTheAnswers(t *testing.T) {
	p := need(t)
	s := sharedSession(t)
	src := filepath.Join(p.checkpoints, "multilingual")
	raw, err := os.ReadFile(filepath.Join(src, "rl_agent_config.json"))
	if err != nil {
		t.Fatal(err)
	}
	sc := singles(t)[0]
	if sc.name != "billing-mixed" {
		t.Fatalf("first case is %s", sc.name)
	}
	answers := func(edit func(cfg map[string]any)) map[string]string {
		var cfg map[string]any
		if err := json.Unmarshal(raw, &cfg); err != nil {
			t.Fatal(err)
		}
		edit(cfg)
		dir := t.TempDir()
		entries, err := os.ReadDir(src)
		if err != nil {
			t.Fatal(err)
		}
		for _, e := range entries {
			if e.Name() != "rl_agent_config.json" {
				if err := os.Symlink(filepath.Join(src, e.Name()), filepath.Join(dir, e.Name())); err != nil {
					t.Fatal(err)
				}
			}
		}
		data, _ := json.Marshal(cfg)
		if err := os.WriteFile(filepath.Join(dir, "rl_agent_config.json"), data, 0o644); err != nil {
			t.Fatal(err)
		}
		a, err := agent.Open(os.DirFS(dir), keepOpen{s})
		if err != nil {
			t.Fatal(err)
		}
		defer a.Close()
		pred, err := a.Predict(context.Background(), sc.state, sc.qs)
		if err != nil {
			t.Fatal(err)
		}
		data, _ = json.Marshal(pred)
		parsed, err := fixtures.Parse(data)
		if err != nil {
			t.Fatal(err)
		}
		out := map[string]string{}
		for _, m := range fixtures.Obj(t, fixtures.Field(t, parsed, "answers")) {
			out[m.Key] = string(fixtures.Marshal(t, m.Value))
		}
		return out
	}
	base := answers(func(map[string]any) {})
	if len(base) != 3 {
		t.Fatalf("base answers: %v", base)
	}
	ids := map[string]string{"choice": "intent", "score": "urgency", "noul": "duplicate"}
	for i, typ := range []string{"choice", "score", "noul"} {
		temps := []any{1.0, 1.0, 1.0}
		temps[i] = 4.0
		got := answers(func(c map[string]any) { c["temperature"] = temps })
		for _, other := range []string{"choice", "score", "noul"} {
			changed := got[ids[other]] != base[ids[other]]
			if changed != (other == typ) {
				t.Errorf("temperature %v: %s answer changed = %v", temps, other, changed)
			}
		}
	}
	got := answers(func(c map[string]any) { c["temperature_by_options"] = map[string]any{"choice:3-5": 4.0} })
	if got["intent"] == base["intent"] || got["urgency"] != base["urgency"] || got["duplicate"] != base["duplicate"] {
		t.Errorf("a choice:3-5 bucket temperature did not change exactly the 3-option choice: %v", got)
	}
}