Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

25.7 KB
Raw Blame
package decode

import (
	"encoding/json"
	"errors"
	"math"
	"strconv"
	"strings"
	"testing"

	"github.com/NandhaKishorM/laya/laya-go/internal/fixtures"
	"github.com/NandhaKishorM/laya/laya-go/internal/goldens"
	"github.com/NandhaKishorM/laya/laya-go/internal/pyjson"
	"github.com/NandhaKishorM/laya/laya-go/internal/pyvalue"
	"github.com/NandhaKishorM/laya/laya-go/schema"
)

// plain converts a parsed golden value into the maps and slices a JSON config decodes to.
func plain(v any) any {
	switch x := v.(type) {
	case fixtures.Object:
		out := make(map[string]any, len(x))
		for _, m := range x {
			out[m.Key] = plain(m.Value)
		}
		return out
	case []any:
		out := make([]any, len(x))
		for i, item := range x {
			out[i] = plain(item)
		}
		return out
	}
	return v
}

func floats32(tb testing.TB, v any) []float32 {
	tb.Helper()
	list := fixtures.List(tb, v)
	out := make([]float32, len(list))
	for i, item := range list {
		out[i] = float32(fixtures.Float(tb, item))
	}
	return out
}

// TestDecodeGolden holds the decoder to Agent._decode_answers: the real method, on logit rows
// aimed at every temperature bucket, a tie, a uniform row, overflowing magnitudes, the option
// permutation and the language override. The answer is compared as the JSON it marshals to, so
// key order, rounding and float spelling are all part of the contract.
func TestDecodeGolden(t *testing.T) {
	golden := fixtures.Load(t, "decode.json")
	for _, name := range []string{"english", "multilingual"} {
		entry := fixtures.Obj(t, fixtures.Field(t, golden, name))
		config := fixtures.Obj(t, entry.Must("config"))
		temps, err := NewTemperatures(
			fixtures.List(t, config.Must("temperature")),
			plain(config.Must("temperature_by_options")).(map[string]any),
			plain(config.Must("lang_temperatures_raw")).(map[string]any))
		if err != nil {
			t.Fatalf("%s: %v", name, err)
		}
		cases := fixtures.Obj(t, entry.Must("cases"))
		if len(cases) < 20 {
			t.Fatalf("%s: the golden holds %d cases", name, len(cases))
		}
		for _, c := range cases {
			rec := fixtures.Obj(t, c.Value)
			t.Run(name+"/"+c.Key, func(t *testing.T) {
				var order []int32
				if o := rec.Must("option_order"); o != nil {
					order = fixtures.Ints(t, o)
				}
				q := goldens.Question(t, fixtures.Obj(t, rec.Must("question")), order)
				lang := fixtures.StrOrEmpty(t, rec.Must("lang"))
				logits := floats32(t, rec.Must("logits"))
				k := len(logits)
				width := fixtures.Int(t, rec.Must("logits_block_width"))
				for len(logits) < width { // the columns past k hold 99, which a decoder must not read
					logits = append(logits, 99)
				}
				row := Row{Logits: logits, K: k, Act: floats32(t, rec.Must("act"))}
				answer, err := Decode(q, row, lang, temps)
				if err != nil {
					t.Fatal(err)
				}
				got, err := answer.(interface{ MarshalJSON() ([]byte, error) }).MarshalJSON()
				if err != nil {
					t.Fatal(err)
				}
				if want := fixtures.Marshal(t, rec.Must("answer")); string(got) != string(want) {
					t.Errorf("answer differs from Python's\n got %s\nwant %s", got, want)
				}
			})
		}
	}
}

// TestTemperaturesGolden resolves the checkpoints' own configs, as the agent does: the raw
// tables from sequences.json, clamped, must be the tables decode.json was recorded with.
func TestTemperaturesGolden(t *testing.T) {
	seqs := fixtures.Load(t, "sequences.json")
	golden := fixtures.Load(t, "decode.json")
	for _, name := range []string{"english", "multilingual"} {
		raw := fixtures.Obj(t, fixtures.Obj(t, fixtures.Field(t, seqs, name)).Must("config"))
		recorded := fixtures.Obj(t, fixtures.Obj(t, fixtures.Field(t, golden, name)).Must("config"))
		byOptions, _ := plain(raw.Must("temperature_by_options")).(map[string]any)
		temps, err := NewTemperatures(fixtures.List(t, raw.Must("temperature")), byOptions, nil)
		if err != nil {
			t.Fatal(err)
		}
		for i, v := range fixtures.List(t, recorded.Must("temperature")) {
			if want := fixtures.Float(t, v); temps.ByType[i] != want {
				t.Errorf("%s: ByType[%d] = %v, Python clamps to %v", name, i, temps.ByType[i], want)
			}
		}
		wantBy := fixtures.Obj(t, recorded.Must("temperature_by_options"))
		if len(temps.ByOptions) != len(wantBy) {
			t.Errorf("%s: %d bucket temperatures, Python has %d", name, len(temps.ByOptions), len(wantBy))
		}
		for _, m := range wantBy {
			if got, want := temps.ByOptions[m.Key], fixtures.Float(t, m.Value); got != want {
				t.Errorf("%s: %s = %v, Python clamps to %v", name, m.Key, got, want)
			}
		}
	}
}

func TestRound4Golden(t *testing.T) {
	golden := fixtures.Obj(t, fixtures.Field(t, fixtures.Load(t, "go_extra.json"), "round4"))
	values, rounded := fixtures.List(t, golden.Must("values")), fixtures.List(t, golden.Must("rounded"))
	if len(values) < 5000 || len(values) != len(rounded) {
		t.Fatalf("golden holds %d values and %d results", len(values), len(rounded))
	}
	bad := 0
	for i := range values {
		v, want := fixtures.Float(t, values[i]), fixtures.Float(t, rounded[i])
		got := Round4(v)
		if math.Float64bits(got) != math.Float64bits(want) {
			if bad++; bad <= 5 {
				t.Errorf("Round4(%v) = %v, Python round gives %v", v, got, want)
			}
		}
	}
	if bad > 5 {
		t.Errorf("%d of %d values differ", bad, len(values))
	}
}

func TestRound4Edges(t *testing.T) {
	cases := map[string]struct{ in, want float64 }{
		"tie rounds down to even": {0.03125, 0.0312},
		"tie rounds up to even":   {0.09375, 0.0938},
		"negative zero kept":      {math.Copysign(0, -1), math.Copysign(0, -1)},
		"tiny negative is -0":     {-1e-5, math.Copysign(0, -1)},
		"already four places":     {0.1234, 0.1234},
	}
	for name, c := range cases {
		if got := Round4(c.in); math.Float64bits(got) != math.Float64bits(c.want) {
			t.Errorf("%s: Round4(%v) = %v, want %v", name, c.in, got, c.want)
		}
	}
	for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} {
		if got := Round4(v); !(got == v || (math.IsNaN(got) && math.IsNaN(v))) {
			t.Errorf("Round4(%v) = %v, want it passed through", v, got)
		}
	}
}

func TestClamp(t *testing.T) {
	cases := []struct {
		name string
		in   any
		want float64
	}{
		{"inside", 2.0, 2.0}, {"below is raised", 0.1006, 0.5}, {"above is lowered", 9.0, 5.0},
		{"bool is not a number", true, 1}, {"false is not a number", false, 1},
		{"nil", nil, 1}, {"NaN", math.NaN(), 1}, {"+Inf", math.Inf(1), 1}, {"-Inf", math.Inf(-1), 1},
		{"integer", 3, 3}, {"small integer raised", 0, 0.5}, {"numeric string", "2.5", 2.5},
		{"padded string", " 2.5 ", 2.5}, {"text", "warm", 1}, {"hex float is not Python", "0x1p1", 1},
		{"list", []any{1.0}, 1}, {"float32", float32(1.5), 1.5},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := Clamp(c.in); got != c.want {
				t.Errorf("Clamp(%v) = %v, want %v", c.in, got, c.want)
			}
		})
	}
}

func TestBucket(t *testing.T) {
	cases := []struct {
		typ  schema.Type
		k    int
		want string
	}{
		{schema.TypeChoice, 0, "choice:2"}, {schema.TypeChoice, 2, "choice:2"}, {schema.TypeChoice, 3, "choice:3-5"},
		{schema.TypeScore, 5, "score:3-5"}, {schema.TypeScore, 6, "score:6-10"}, {schema.TypeNoul, 10, "noul:6-10"},
		{schema.TypeChoice, 11, "choice:11+"}, {schema.TypeChoice, 400, "choice:11+"},
	}
	for _, c := range cases {
		if got := Bucket(c.typ, c.k); got != c.want {
			t.Errorf("Bucket(%s, %d) = %s, want %s", c.typ, c.k, got, c.want)
		}
	}
}

func TestTemperaturesFor(t *testing.T) {
	temps, err := NewTemperatures([]any{2.0, 3.0, 4.0}, map[string]any{"choice:3-5": 1.5, "noul:2": 0.1},
		map[string]any{
			"ZH-hans": map[string]any{"temperature": []any{1.1, 1.2, 1.3}, "temperature_by_options": map[string]any{"choice:2": 9}},
			"de":      nil,
			"fr":      map[string]any{"temperature": nil},
		})
	if err != nil {
		t.Fatal(err)
	}
	cases := []struct {
		name     string
		typ      schema.Type
		k        int
		lang     string
		want     float64
		override bool
	}{
		{"type default", schema.TypeScore, 4, "", 3, false},
		{"bucket beats type", schema.TypeChoice, 4, "", 1.5, false},
		{"bucket value is clamped", schema.TypeNoul, 2, "", 0.5, false},
		{"language replaces both", schema.TypeChoice, 4, "zh", 1.1, true},
		{"language subtag and case", schema.TypeScore, 4, "ZH-Hant-TW", 1.2, true},
		{"language bucket clamped", schema.TypeChoice, 2, "zh", 5, true},
		{"unknown language falls through", schema.TypeChoice, 4, "xx", 1.5, false},
		{"null override inherits the raw table", schema.TypeScore, 4, "de", 3, true},
		{"null temperature inherits", schema.TypeNoul, 4, "fr", 4, true},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got, over := temps.For(c.typ, c.k, c.lang)
			if got != c.want || over != c.override {
				t.Errorf("For(%s, %d, %q) = %v, %v; want %v, %v", c.typ, c.k, c.lang, got, over, c.want, c.override)
			}
		})
	}
	if got, over := (*Temperatures)(nil).For(schema.TypeChoice, 3, "zh"); got != 1 || over {
		t.Errorf("a nil table gives %v, %v", got, over)
	}
}

func TestNewTemperaturesRefuses(t *testing.T) {
	cases := map[string]struct {
		byType []any
		langs  map[string]any
		want   string
	}{
		"two temperatures":         {[]any{1.0, 1.0}, nil, "3 values"},
		"language not a mapping":   {nil, map[string]any{"de": 5}, "must be a mapping"},
		"language list too short":  {nil, map[string]any{"de": map[string]any{"temperature": []any{1.0}}}, "list of 3"},
		"language list not a list": {nil, map[string]any{"de": map[string]any{"temperature": 2}}, "list of 3"},
		"by options not a mapping": {nil, map[string]any{"de": map[string]any{"temperature_by_options": []any{1.0}}}, "mapping of bucket"},
	}
	for name, c := range cases {
		_, err := NewTemperatures(c.byType, nil, c.langs)
		if err == nil || !strings.Contains(err.Error(), c.want) {
			t.Errorf("%s: err = %v, want it to mention %q", name, err, c.want)
		}
	}
}

func mustQ(q schema.Question, err error) schema.Question { return schema.MustQuestion(q, err) }

func TestDecodeBranches(t *testing.T) {
	two := mustQ(schema.ChoiceOf("pick", "a", "b"))
	temps := &Temperatures{ByType: [3]float64{1, 1, 1}}
	act := []float32{0.25}

	t.Run("binning applies without a language override only", func(t *testing.T) {
		binned := &Temperatures{ByType: [3]float64{1, 1, 1},
			Binning:    map[string]Bins{"choice:2": {Values: []float64{0.1, 0.2, 0.3, 0.9}}},
			ByLanguage: map[string]LanguageTemperatures{"zh": {ByType: [3]float64{1, 1, 1}}}}
		row := Row{Logits: []float32{3, 0}, K: 2, Act: act} // max(p) = 0.9526 -> bin 3
		a, err := Decode(two, row, "", binned)
		if err != nil {
			t.Fatal(err)
		}
		if got := a.Base().AnswerConfidence; got != 0.9 {
			t.Errorf("binned AnswerConfidence = %v, want 0.9", got)
		}
		a, _ = Decode(two, row, "zh", binned)
		if got := a.Base().AnswerConfidence; got != 0.9526 {
			t.Errorf("with a language override AnswerConfidence = %v, want the raw 0.9526", got)
		}
		other := &Temperatures{ByType: [3]float64{1, 1, 1}, Binning: map[string]Bins{"score:2": {Values: []float64{0.1}}}}
		a, _ = Decode(two, row, "", other)
		if got := a.Base().AnswerConfidence; got != 0.9526 {
			t.Errorf("a bucket the map was not fitted for = %v, want the raw 0.9526", got)
		}
	})

	t.Run("a single option has confidence one", func(t *testing.T) {
		one := mustQ(schema.ChoiceOf("pick", "only"))
		a, err := Decode(one, Row{Logits: []float32{7}, K: 1, Act: act}, "", temps)
		if err != nil {
			t.Fatal(err)
		}
		c := a.(*schema.ChoiceAnswer)
		if c.Choice != schema.Text("only") || c.Confidence != 1 || c.AnswerConfidence != 1 {
			t.Errorf("single option answer = %+v", c)
		}
	})

	t.Run("score reports the expected level", func(t *testing.T) {
		s := mustQ(schema.Score("rate", "bad", "ok", "good"))
		a, err := Decode(s, Row{Logits: []float32{0, 0, 0}, K: 3, Act: act}, "", temps)
		if err != nil {
			t.Fatal(err)
		}
		if got := a.(*schema.ScoreAnswer).Score; got != 1 {
			t.Errorf("uniform three levels score %v, want 1", got)
		}
	})

	t.Run("first maximum wins a tie", func(t *testing.T) {
		a, _ := Decode(two, Row{Logits: []float32{1, 1}, K: 2, Act: act}, "", temps)
		if got := a.(*schema.ChoiceAnswer).Choice; got != schema.Text("a") {
			t.Errorf("tie chose %q", got)
		}
	})

	t.Run("a permuted row is put back in label order", func(t *testing.T) {
		q, err := mustQ(schema.ChoiceOf("pick", "a", "b", "c")).WithOptionOrder([]int{2, 0, 1})
		if err != nil {
			t.Fatal(err)
		}
		// slot 0 shows option 2 and carries the largest logit, so option "c" wins
		a, _ := Decode(q, Row{Logits: []float32{5, 0, 0}, K: 3, Act: act}, "", temps)
		if got := a.(*schema.ChoiceAnswer).Choice; got != schema.Text("c") {
			t.Errorf("permuted decode chose %q, want c", got)
		}
	})
}

func TestDecodeRefuses(t *testing.T) {
	q := mustQ(schema.ChoiceOf("pick", "a", "b"))
	noul := mustQ(schema.Noul("holds"))
	cases := map[string]struct {
		q   schema.Question
		row Row
	}{
		"zero question":       {schema.Question{}, Row{Logits: []float32{1, 2}, K: 2, Act: []float32{0}}},
		"wrong marker count":  {q, Row{Logits: []float32{1, 2, 3}, K: 3, Act: []float32{0}}},
		"row too narrow":      {q, Row{Logits: []float32{1}, K: 2, Act: []float32{0}}},
		"no action":           {q, Row{Logits: []float32{1, 2}, K: 2}},
		"NaN logit":           {q, Row{Logits: []float32{float32(math.NaN()), 2}, K: 2, Act: []float32{0}}},
		"infinite logit":      {q, Row{Logits: []float32{float32(math.Inf(1)), 2}, K: 2, Act: []float32{0}}},
		"NaN action":          {q, Row{Logits: []float32{1, 2}, K: 2, Act: []float32{float32(math.NaN())}}},
		"noul row too narrow": {noul, Row{Logits: []float32{1}, K: 1, Act: []float32{0}}},
	}
	for name, c := range cases {
		t.Run(name, func(t *testing.T) {
			if _, err := Decode(c.q, c.row, "", nil); err == nil {
				t.Fatal("no error")
			} else if !strings.Contains(err.Error(), ErrShape.Error()) {
				t.Errorf("err = %v, want it to wrap ErrShape", err)
			}
		})
	}
}

func TestActionProbabilities(t *testing.T) {
	got, err := ActionProbabilities([]float32{0, 0})
	if err != nil || len(got) != 2 || got[0] != 0.5 || got[1] != 0.5 {
		t.Errorf("uniform = %v, %v", got, err)
	}
	got, err = ActionProbabilities([]float32{1000, 0})
	if err != nil || got[0] != 1 || got[1] != 0 {
		t.Errorf("a large logit overflowed: %v, %v", got, err)
	}
	for name, row := range map[string][]float32{"empty": nil, "NaN": {float32(math.NaN())}} {
		if _, err := ActionProbabilities(row); err == nil {
			t.Errorf("%s row: no error", name)
		}
	}
}

func fixtureQuestion(t *testing.T, kind string, crit any, order []int) schema.Question {
	t.Helper()
	var (
		q   schema.Question
		err error
	)
	switch kind {
	case "choice":
		labels := []string{}
		for _, l := range crit.([]any) {
			labels = append(labels, l.(string))
		}
		q, err = schema.ChoiceOf("i", labels...)
	case "score":
		levels := crit.([]any)
		q, err = schema.Score("i", levels...)
	default:
		q, err = schema.Noul("i")
	}
	if err != nil {
		t.Fatal(err)
	}
	if order != nil {
		if q, err = q.WithOptionOrder(order); err != nil {
			t.Fatal(err)
		}
	}
	return q
}

func orderOf(t *testing.T, v any) []int {
	t.Helper()
	if v == nil {
		return nil
	}
	var out []int
	for _, i := range fixtures.Ints(t, v) {
		out = append(out, int(i))
	}
	return out
}

// TestLabelAnswersGolden holds the answer JSON to the bytes Python writes for bool, int, float and
// str labels: "choice": true, "choice": 1, the key "Infinity", the key "true". A label's type
// survives the answer, and 1 and "1" stay two labels.
func TestLabelAnswersGolden(t *testing.T) {
	cases := fixtures.List(t, fixtures.Obj(t, fixtures.Field(t, fixtures.Load(t, "go_extra.json"), "label_answers")).Must("cases"))
	if len(cases) < 400 {
		t.Fatalf("the golden holds %d cases", len(cases))
	}
	typed := 0
	for i, c := range cases {
		rec := fixtures.Obj(t, c)
		criteria := fixtures.Str(t, rec.Must("criteria_text"))
		qs, err := schema.ParseQuestions([]byte(`{"q":{"type":"choice","instructions":"i","criteria":` + criteria + `}}`))
		if err != nil {
			t.Fatalf("case %d %s: %v", i, criteria, err)
		}
		q, _ := qs.Get("q")
		if order := orderOf(t, rec.Must("option_order")); order != nil {
			if q, err = q.WithOptionOrder(order); err != nil {
				t.Fatal(err)
			}
		}
		if got, want := q.RenderedOptions(), fixtures.Strings(t, rec.Must("options")); strings.Join(got, "\x00") != strings.Join(want, "\x00") {
			t.Errorf("case %d %s: options %q, want %q", i, criteria, got, want)
		}
		logits := floats32(t, rec.Must("logits"))
		answer, err := Decode(q, Row{Logits: logits, K: len(logits), Act: []float32{0.25}}, "", &Temperatures{ByType: [3]float64{1, 1, 1}})
		if err != nil {
			t.Fatal(err)
		}
		got, err := answer.(json.Marshaler).MarshalJSON()
		if err != nil {
			t.Fatal(err)
		}
		if want := fixtures.Str(t, rec.Must("answer_text")); string(got) != want {
			t.Errorf("case %d %s\n got %s\nwant %s", i, criteria, got, want)
		}
		if !strings.HasPrefix(string(got), `{"type":"choice","choice":"`) {
			typed++
		}
	}
	if typed < 200 {
		t.Errorf("only %d answers have a non-string choice", typed)
	}
}

// TestDecodeFuzzGolden decodes 5,000 random, tied and near-tied rows and compares the marshaled
// answer with Python's bytes. Two kinds of difference are tolerated and counted, because Decode
// computes in float64 where numpy computes in float32, and numpy's float32 exp is a SIMD routine
// whose last bits differ between CPUs (the same row is answered differently by Python on arm64
// and on x86-64):
//
//   - a number that differs by one unit in the fourth decimal (a value within 1e-4 sitting on a
//     rounding boundary);
//   - the choice, when the option Python chose and the one Go chose both have a quotient within
//     nearTieBand of the largest. Go's is the first maximum of the float64 probabilities;
//     everything else in the answer must still be Python's.
//
// Anything else fails.
func TestDecodeFuzzGolden(t *testing.T) {
	cases := fixtures.List(t, fixtures.Obj(t, fixtures.Field(t, fixtures.Load(t, "go_extra.json"), "answers_fuzz")).Must("cases"))
	if len(cases) != 5000 {
		t.Fatalf("the golden holds %d cases", len(cases))
	}
	identical, nearTie, rounded, bad := 0, 0, 0, 0
	for i, c := range cases {
		rec := fixtures.Obj(t, c)
		kind := fixtures.Str(t, rec.Must("type"))
		order := orderOf(t, rec.Must("option_order"))
		q := fixtureQuestion(t, kind, rec.Must("criteria"), order)
		var logits []float32
		for _, s := range fixtures.Strings(t, rec.Must("logits")) {
			f, err := strconv.ParseFloat(s, 32)
			if err != nil {
				t.Fatal(err)
			}
			logits = append(logits, float32(f))
		}
		temp := rec.Must("temperature")
		temps, err := NewTemperatures([]any{temp, temp, temp}, nil, nil)
		if err != nil {
			t.Fatal(err)
		}
		answer, err := Decode(q, Row{Logits: logits, K: len(logits), Act: []float32{0.25}}, "", temps)
		if err != nil {
			t.Fatal(err)
		}
		out, _ := answer.(json.Marshaler).MarshalJSON()
		want := fixtures.Str(t, rec.Must("answer_text"))
		if string(out) == want {
			identical++
			continue
		}
		gotTree, _ := pyvalue.Parse(out)
		wantTree, _ := pyvalue.Parse([]byte(want))
		var diffs []jsonDiff
		diffJSON("", gotTree, wantTree, &diffs)
		ok, sawRound, sawChoice := true, false, false
		for _, d := range diffs {
			switch {
			case d.path == "/choice" && kind == "choice":
				goIdx, _ := strconv.Atoi(d.got[2:])
				pyIdx, _ := strconv.Atoi(d.want[2:])
				sawChoice = true
				ok = ok && nearTied(logits, order, temps, kind, goIdx, pyIdx)
			case d.delta > 0 && d.delta < 1.5e-4:
				sawRound = true
			default:
				ok = false
			}
		}
		switch {
		case !ok:
			if bad++; bad <= 5 {
				t.Errorf("case %d (%s, k=%d, T=%v, logits %v)\n go: %s\n py: %s", i, kind, len(logits), temp, logits, out, want)
			}
		case sawChoice:
			nearTie++
		case sawRound:
			rounded++
		}
	}
	t.Logf("%d of %d decodes are Python's bytes; %d differ only in the choice among float32 near-ties; %d only by 1e-4 on a rounding boundary; %d differ otherwise",
		identical, len(cases), nearTie, rounded, bad)
	if bad > 0 {
		t.Errorf("%d of %d decodes differ from Python beyond the tolerated float32 effects", bad, len(cases))
	}
	if rounded > 25 {
		t.Errorf("%d rounding rows: the tolerance is hiding more than float32 arithmetic", rounded)
	}
}

type jsonDiff struct {
	path, got, want string
	delta           float64
}

// diffJSON lists where two decoded answers differ; numbers carry their distance.
func diffJSON(path string, got, want any, out *[]jsonDiff) {
	switch w := want.(type) {
	case pyjson.Object:
		g, ok := got.(pyjson.Object)
		if !ok || len(g) != len(w) {
			*out = append(*out, jsonDiff{path: path, got: pyvalue.Repr(got), want: pyvalue.Repr(want), delta: -1})
			return
		}
		for i := range w {
			if g[i].Key != w[i].Key {
				*out = append(*out, jsonDiff{path: path, got: pyvalue.Repr(got), want: pyvalue.Repr(want), delta: -1})
				return
			}
			diffJSON(path+"/"+w[i].Key.(string), g[i].Value, w[i].Value, out)
		}
	case json.Number:
		g, ok := got.(json.Number)
		if !ok {
			*out = append(*out, jsonDiff{path: path, got: pyvalue.Repr(got), want: pyvalue.Repr(want), delta: -1})
			return
		}
		if g != w {
			a, _ := g.Float64()
			b, _ := w.Float64()
			*out = append(*out, jsonDiff{path: path, got: string(g), want: string(w), delta: math.Abs(a - b)})
		}
	default:
		if pyvalue.Repr(got) != pyvalue.Repr(want) {
			*out = append(*out, jsonDiff{path: path, got: pyvalue.Repr(got), want: pyvalue.Repr(want), delta: -1})
		}
	}
}

// nearTieBand is how far below the largest quotient an option may sit and still be one Python can
// pick: about sixteen float32 steps at 1, a probability ratio of 1 - 1e-6.
const nearTieBand = 1e-6

// nearTied reports whether the options Go and Python chose both sit within nearTieBand of the
// largest quotient.
func nearTied(logits []float32, order []int, temps *Temperatures, kind string, goIdx, pyIdx int) bool {
	t, _ := temps.For(schema.Type(kind), len(logits), "")
	z := make([]float32, len(logits))
	top := float32(math.Inf(-1))
	for i, l := range logits {
		z[i] = l / float32(t)
		top = max(top, z[i])
	}
	if order != nil {
		canonical := make([]float32, len(z))
		for slot, option := range order {
			canonical[option] = z[slot]
		}
		z = canonical
	}
	for _, i := range []int{goIdx, pyIdx} {
		if i < 0 || i >= len(z) || float64(top)-float64(z[i]) > nearTieBand {
			return false
		}
	}
	return true
}

// TestQuotientTieFallsToTheFirst: Python divides the float32 logits by the temperature in float32,
// so two different logits can become one quotient, an exact tie that numpy.argmax gives to the
// lower index. Each pair is two adjacent float32 values that tie at its temperature, and numpy
// answers option 0 for both. The first pair fails if the quotient is kept in float64; the second,
// whose temperature is not a float32, also fails if the division is done in float64 and rounded.
func TestQuotientTieFallsToTheFirst(t *testing.T) {
	q, err := schema.ChoiceOf("pick", "first", "second")
	if err != nil {
		t.Fatal(err)
	}
	for _, c := range []struct {
		temperature  float64
		lower, upper uint32
	}{
		{3.0, 1069547521, 1069547522},
		{1.3, 1072020843, 1072020844},
	} {
		lower, upper := math.Float32frombits(c.lower), math.Float32frombits(c.upper)
		if !(lower < upper) || lower/float32(c.temperature) != upper/float32(c.temperature) {
			t.Fatalf("%v and %v no longer tie when divided by %v in float32", lower, upper, c.temperature)
		}
		temps, err := NewTemperatures([]any{c.temperature, c.temperature, c.temperature}, nil, nil)
		if err != nil {
			t.Fatal(err)
		}
		answer, err := Decode(q, Row{Logits: []float32{lower, upper}, K: 2, Act: []float32{0.5}}, "", temps)
		if err != nil {
			t.Fatal(err)
		}
		if got := answer.(*schema.ChoiceAnswer).Choice; got != schema.Text("first") {
			t.Errorf("T=%v: choice = %v, want first: the quotients are one float32", c.temperature, got)
		}
	}
}

// TestLargeLogitsDoNotOverflow: the maximum is subtracted before exp, so a logit of 800, whose
// exp is not a float64, still answers 1.0 and 0.0 as Python does.
func TestLargeLogitsDoNotOverflow(t *testing.T) {
	q, err := schema.ChoiceOf("pick", "first", "second")
	if err != nil {
		t.Fatal(err)
	}
	temps, err := NewTemperatures([]any{1.0, 1.0, 1.0}, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	answer, err := Decode(q, Row{Logits: []float32{800, 0}, K: 2, Act: []float32{0.5}}, "", temps)
	if err != nil {
		t.Fatal(err)
	}
	out, err := answer.(json.Marshaler).MarshalJSON()
	if err != nil {
		t.Fatal(err)
	}
	const want = `{"type":"choice","choice":"first","probabilities":{"first":1.0,"second":0.0},"confidence":1.0,"answer_confidence":1.0,"action":{"act_probability":0.5}}`
	if string(out) != want {
		t.Errorf("got  %s\nwant %s", out, want)
	}
}

// TestQuotientOverflowIsRefused: a finite logit can overflow float32 when divided by a
// temperature below 1. Python answers NaN for every field; Decode refuses the row. A quotient
// that overflows downwards is no trouble on either side: its probability is 0.
func TestQuotientOverflowIsRefused(t *testing.T) {
	q, err := schema.ChoiceOf("pick", "first", "second")
	if err != nil {
		t.Fatal(err)
	}
	temps, err := NewTemperatures([]any{0.5, 0.5, 0.5}, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if answer, err := Decode(q, Row{Logits: []float32{3e38, 1}, K: 2, Act: []float32{0.5}}, "", temps); !errors.Is(err, ErrShape) {
		t.Errorf("answer %v, error %v; want ErrShape", answer, err)
	}
	answer, err := Decode(q, Row{Logits: []float32{1, -3e38}, K: 2, Act: []float32{0.5}}, "", temps)
	if err != nil {
		t.Fatal(err)
	}
	if ps := answer.(*schema.ChoiceAnswer).Probabilities; ps[0].P != 1 || ps[1].P != 0 {
		t.Errorf("probabilities = %v, want 1 and 0", ps)
	}
}

func TestNewTemperaturesFalsyByOptionsIsEmpty(t *testing.T) {
	for _, v := range []any{false, 0, 0.0, "", []any{}, map[string]any{}, nil, json.Number("0"), json.Number("0.0")} {
		if _, err := NewTemperatures(nil, nil, map[string]any{"de": map[string]any{"temperature_by_options": v}}); err != nil {
			t.Errorf("%#v: %v", v, err)
		}
	}
}

func TestNewTemperaturesRefusesSharedPrimarySubtag(t *testing.T) {
	_, err := NewTemperatures(nil, nil, map[string]any{"PT": map[string]any{}, "pt-BR": map[string]any{}})
	if err == nil || !strings.Contains(err.Error(), `"PT"`) || !strings.Contains(err.Error(), `"pt-BR"`) {
		t.Fatalf("err = %v", err)
	}
}