Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(go): add laya-go/onnx, a nested module that runs the graph with ONNX Runtime
Adeshina Lasisi39c811532d70
diff --git a/laya-go/README.md b/laya-go/README.mdindex 43509b8f..16f97e20 100644--- a/laya-go/README.md+++ b/laya-go/README.md@@ -12,6 +12,7 @@ The foundation (language detection, questions, sequences, the tokenizer) and the| `sequence` | `build_sequence`, `build_head`, `state_room`, `window_budget` and `window_batch_cap`: `EncodeState`, `Build`, `StateRoom`, `PlanWindows`, `BatchCap`. || `infer` | The seam to a model runtime: `Batch` (the graph's five inputs), `Output` (its two), `Session`, and the shape checks `Batch.Validate` and `Output.Check`. || `agent` | `ONNXAgent.predict` and `predict_batch` over an `infer.Session`: `Open`, `Predict`, `PredictBatch`, `LoadConfig`, with batch size, sort-by-length and context cancellation between runs. |+| `onnx` | A nested module (cgo, its own `go.mod`): `onnx.Open` runs a fused `laya.onnx` on ONNX Runtime and implements `infer.Session`, for `agent.Open`. Needs an ONNX Runtime 1.29+ shared library; `CGO_ENABLED=0` builds and `Open` returns `ErrNoCGO`. || `tokenizer` | A reader and runner for Hugging Face `tokenizer.json` (BPE): normalizer, pre-tokenizer, added tokens, byte fallback, decoder. || `internal/unicodedata` | The character predicates `laya/lang.py` leans on, recorded from CPython. || `internal/pyjson` | `json.dumps` and `repr` spelled as CPython spells them. |diff --git a/laya-go/onnx/cancel_test.go b/laya-go/onnx/cancel_test.gonew file mode 100644index 00000000..d7287f6a--- /dev/null+++ b/laya-go/onnx/cancel_test.go@@ -0,0 +1,127 @@+//go:build cgo++package onnx++import (+ "context"+ "errors"+ "runtime"+ "testing"+ "time"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++func TestAlreadyCancelledContextDoesNotRun(t *testing.T) {+ s := sharedSession(t)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ before := s.runs.Load()+ _, err := s.Run(ctx, makeBatch(2, 12, 3))+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Run = %v, want context.Canceled", err)+ }+ dctx, dcancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))+ defer dcancel()+ if _, err := s.Run(dctx, makeBatch(2, 12, 3)); !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("Run with an expired deadline = %v", err)+ }+ if n := s.runs.Load() - before; n != 0 {+ t.Fatalf("the model ran %d times for cancelled contexts", n)+ }+}++func TestInvalidBatchIsRefusedBeforeRunning(t *testing.T) {+ s := sharedSession(t)+ before := s.runs.Load()+ b := makeBatch(2, 12, 3)+ b.QType = b.QType[:1]+ var se *infer.ShapeError+ if _, err := s.Run(context.Background(), b); !errors.As(err, &se) || se.Name != infer.NameQType {+ t.Fatalf("Run = %v, want a ShapeError for qtype", err)+ }+ if s.runs.Load() != before {+ t.Fatal("the model ran on an invalid batch")+ }+ out, err := s.Run(context.Background(), infer.Batch{})+ if err != nil || len(out.Logits) != 0 || s.runs.Load() != before {+ t.Fatalf("empty batch: %v, %v", out, err)+ }+}++func TestCancelMidRun(t *testing.T) {+ s := sharedSession(t)+ b := makeBatch(24, 256, 4)+ start := time.Now()+ if _, err := s.Run(context.Background(), b); err != nil {+ t.Fatal(err)+ }+ full := time.Since(start)+ t.Logf("uncancelled run of the large batch: %v", full)++ ctx, cancel := context.WithCancel(context.Background())+ s.runs.Store(0)+ go func() {+ for s.runs.Load() == 0 {+ runtime.Gosched()+ }+ time.Sleep(full / 10)+ cancel()+ }()+ start = time.Now()+ _, err := s.Run(ctx, b)+ took := time.Since(start)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Run = %v after %v, want context.Canceled", err, took)+ }+ if took > full/2 {+ t.Fatalf("a run cancelled at 10%% took %v of %v: the terminate flag did not stop it", took, full)+ }+ t.Logf("cancelled run returned after %v", took)+ // The session is still usable, and a fresh context is not poisoned by the old flag.+ mustRun(t, s)+}++func TestNoGoroutineLeak(t *testing.T) {+ s := sharedSession(t)+ small := makeBatch(2, 16, 3)+ mustRun(t, s)+ settle := func() int {+ for i := 0; i < 100; i++ {+ runtime.GC()+ time.Sleep(10 * time.Millisecond)+ }+ return runtime.NumGoroutine()+ }+ baseline := settle()+ for i := 0; i < 50; i++ {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ if _, err := s.Run(ctx, small); !errors.Is(err, context.Canceled) {+ t.Fatalf("cancelled run %d: %v", i, err)+ }+ }+ for i := 0; i < 50; i++ {+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(i%5)*time.Millisecond)+ _, err := s.Run(ctx, small)+ cancel()+ if err != nil && !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("racing run %d: %v", i, err)+ }+ }+ for i := 0; i < 50; i++ {+ ctx, cancel := context.WithCancel(context.Background())+ if _, err := s.Run(ctx, small); err != nil {+ t.Fatalf("normal run %d: %v", i, err)+ }+ cancel()+ if _, err := s.Run(context.Background(), small); err != nil {+ t.Fatalf("background run %d: %v", i, err)+ }+ }+ after := settle()+ t.Logf("goroutines: baseline %d, after 150 cancellable and 50 plain runs %d", baseline, after)+ if after > baseline {+ t.Fatalf("goroutine count rose from %d to %d", baseline, after)+ }+}diff --git a/laya-go/onnx/doc.go b/laya-go/onnx/doc.gonew file mode 100644index 00000000..d689291c--- /dev/null+++ b/laya-go/onnx/doc.go@@ -0,0 +1,15 @@+// Package onnx runs a laya graph on ONNX Runtime and implements infer.Session, so the pure-Go+// agent package can answer questions with it.+//+// It is a separate module because it needs cgo and a native ONNX Runtime shared library+// (1.29 or newer, which the caller supplies). Open takes the fused laya.onnx, whose .data+// sidecar must sit beside it, and runs it on the CPU provider:+//+// sess, err := onnx.Open("laya.onnx", onnx.WithLibrary("/opt/onnxruntime/libonnxruntime.so.1.30.0"))+// a, err := agent.Open(os.DirFS("checkpoint"), sess)+//+// The library is found through WithLibrary, else $LAYA_ONNXRUNTIME_LIBRARY, else the platform's+// default name on the loader's search path. A process holds one ONNX Runtime: sessions share it,+// and it is unloaded when the last one closes. Built with CGO_ENABLED=0 the package still+// compiles, and Open returns ErrNoCGO.+package onnxdiff --git a/laya-go/onnx/example_test.go b/laya-go/onnx/example_test.gonew file mode 100644index 00000000..c62a900e--- /dev/null+++ b/laya-go/onnx/example_test.go@@ -0,0 +1,35 @@+package onnx_test++import (+ "context"+ "fmt"+ "log"+ "os"++ "github.com/NandhaKishorM/laya/laya-go/agent"+ "github.com/NandhaKishorM/laya/laya-go/onnx"+ "github.com/NandhaKishorM/laya/laya-go/schema"+)++// Open a fused graph, hand it to the agent, and ask one question. The agent closes the session.+func ExampleOpen() {+ sess, err := onnx.Open("laya.onnx", onnx.WithLibrary("/opt/onnxruntime/lib/libonnxruntime.so.1.30.0"))+ if err != nil {+ log.Fatal(err)+ }+ a, err := agent.Open(os.DirFS("checkpoint"), sess)+ if err != nil {+ log.Fatal(err)+ }+ defer a.Close()++ qs, err := schema.ParseQuestions([]byte(`{"urgent": {"type": "noul", "instructions": "Is this urgent?"}}`))+ if err != nil {+ log.Fatal(err)+ }+ pred, err := a.Predict(context.Background(), "The site is down.", qs)+ if err != nil {+ log.Fatal(err)+ }+ fmt.Println(pred.Answers)+}diff --git a/laya-go/onnx/go.mod b/laya-go/onnx/go.modnew file mode 100644index 00000000..a118195d--- /dev/null+++ b/laya-go/onnx/go.mod@@ -0,0 +1,10 @@+module github.com/NandhaKishorM/laya/laya-go/onnx++go 1.22++require (+ github.com/NandhaKishorM/laya/laya-go v0.0.0-00010101000000-000000000000+ github.com/yalue/onnxruntime_go v1.36.0+)++replace github.com/NandhaKishorM/laya/laya-go => ../diff --git a/laya-go/onnx/go.sum b/laya-go/onnx/go.sumnew file mode 100644index 00000000..0b8081bf--- /dev/null+++ b/laya-go/onnx/go.sum@@ -0,0 +1,2 @@+github.com/yalue/onnxruntime_go v1.36.0 h1:iH1Q++DcsyT9sWtN26KYimESlI5hhXpKaChHDS44oV4=+github.com/yalue/onnxruntime_go v1.36.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=diff --git a/laya-go/onnx/graph.go b/laya-go/onnx/graph.gonew file mode 100644index 00000000..4fff6f44--- /dev/null+++ b/laya-go/onnx/graph.go@@ -0,0 +1,164 @@+package onnx++import (+ "errors"+ "fmt"+ "io/fs"+ "os"+ "path/filepath"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++// GraphError reports a graph file laya cannot run: it is missing, is not the fused layout, or+// declares tensors other than the ones the agent feeds.+type GraphError struct {+ // Path is the graph file as passed to Open.+ Path string+ // Problem says what is wrong and, where it helps, how to fix it.+ Problem string+}++// Error names the file and the problem.+func (e *GraphError) Error() string { return fmt.Sprintf("onnx: graph %s: %s", e.Path, e.Problem) }++const reexport = "re-export with scripts/export_onnx.py"++// tensorInfo is what checkGraph needs to know about one declared input or output. Dims holds -1+// for a dynamic dimension.+type tensorInfo struct {+ Name string+ DType string // "int64", "bool", "float32", or whatever else the graph declares+ Dims []int64+}++type tensorSpec struct {+ name, dtype string+ rank int+ // fixedOK is the one fixed size a non-batch dimension may have, or 0 for none.+ fixedOK int64+}++var (+ wantInputs = []tensorSpec{+ {infer.NameInputIDs, "int64", 2, 0}, {infer.NameAttentionMask, "int64", 2, 0},+ {infer.NameMarkerPos, "int64", 2, 0}, {infer.NameMarkerMask, "bool", 2, 0},+ {infer.NameQType, "int64", 1, 0},+ }+ wantOutputs = []tensorSpec{+ {infer.NameLogits, "float32", 2, 0}, {infer.NameActLogits, "float32", 2, 2},+ }+)++// checkGraph verifies the declared tensors are exactly the five inputs and two outputs, with+// the dtypes and ranks the agent feeds, and that every dimension that varies per call is+// dynamic. The returned *GraphError has no Path; Open fills it in.+func checkGraph(inputs, outputs []tensorInfo) error {+ if err := checkSide("input", inputs, wantInputs); err != nil {+ return err+ }+ return checkSide("output", outputs, wantOutputs)+}++func checkSide(kind string, got []tensorInfo, want []tensorSpec) error {+ byName := make(map[string]tensorInfo, len(got))+ for _, g := range got {+ byName[g.Name] = g+ }+ for _, w := range want {+ g, ok := byName[w.name]+ if !ok {+ return &GraphError{Problem: fmt.Sprintf("missing %s %q; %s", kind, w.name, reexport)}+ }+ if g.DType != w.dtype {+ return &GraphError{Problem: fmt.Sprintf("%s %q is %s, want %s; %s", kind, w.name, g.DType, w.dtype, reexport)}+ }+ if len(g.Dims) != w.rank {+ return &GraphError{Problem: fmt.Sprintf("%s %q has rank %d, want %d; %s", kind, w.name, len(g.Dims), w.rank, reexport)}+ }+ for i, d := range g.Dims {+ if d == -1 || (i > 0 && d == w.fixedOK && d > 0) {+ continue+ }+ what := "a fixed size"+ if i == 0 {+ what = "a fixed batch dimension"+ }+ return &GraphError{Problem: fmt.Sprintf("%s %q declares %s %d in shape %v; %s", kind, w.name, what, d, g.Dims, reexport)}+ }+ }+ if len(got) != len(want) {+ for _, g := range got {+ known := false+ for _, w := range want {+ known = known || w.name == g.Name+ }+ if !known {+ return &GraphError{Problem: fmt.Sprintf("unexpected %s %q; the agent feeds only %d inputs and %d outputs; %s", kind, g.Name, len(wantInputs), len(wantOutputs), reexport)}+ }+ }+ }+ return nil+}++// checkPath refuses what is plainly not a fused graph file before any runtime is started.+func checkPath(path string) error {+ info, err := os.Stat(path)+ switch {+ case err == nil && info.IsDir():+ if split(path) {+ return &GraphError{Path: path, Problem: splitProblem}+ }+ return &GraphError{Path: path, Problem: "is a directory; pass the path of laya.onnx"}+ case err == nil:+ if base := filepath.Base(path); (base == "encoder.onnx" || base == "head.onnx") && split(filepath.Dir(path)) {+ return &GraphError{Path: path, Problem: splitProblem}+ }+ return nil+ case errors.Is(err, fs.ErrNotExist):+ if split(filepath.Dir(path)) {+ return &GraphError{Path: path, Problem: "no such file; " + splitProblem}+ }+ return &GraphError{Path: path, Problem: "no such file"}+ }+ return &GraphError{Path: path, Problem: err.Error()}+}++const splitProblem = "the split encoder.onnx + head.onnx layout is not supported; " + reexport + " to write one fused laya.onnx"++// split reports whether dir holds both halves of the split layout.+func split(dir string) bool {+ for _, name := range []string{"encoder.onnx", "head.onnx"} {+ if info, err := os.Stat(filepath.Join(dir, name)); err != nil || info.IsDir() {+ return false+ }+ }+ return true+}++// outTensor is one output as the runtime returns it.+type outTensor struct {+ shape []int64+ data []float32+}++// collect checks both outputs against the batch and copies them into slices the caller owns.+func collect(b infer.Batch, logits, act outTensor) (infer.Output, error) {+ for _, c := range []struct {+ name string+ t outTensor+ cols int+ }{{infer.NameLogits, logits, b.Markers}, {infer.NameActLogits, act, 2}} {+ if len(c.t.shape) != 2 || c.t.shape[0] != int64(b.Rows) || c.t.shape[1] != int64(c.cols) {+ return infer.Output{}, fmt.Errorf("onnx: output %s has shape %v, want [%d %d]", c.name, c.t.shape, b.Rows, c.cols)+ }+ }+ o := infer.Output{+ Logits: append([]float32(nil), logits.data...),+ ActLogits: append([]float32(nil), act.data...),+ }+ if err := o.Check(b); err != nil {+ return infer.Output{}, err+ }+ return o, nil+}diff --git a/laya-go/onnx/graph_test.go b/laya-go/onnx/graph_test.gonew file mode 100644index 00000000..3aa7e5ff--- /dev/null+++ b/laya-go/onnx/graph_test.go@@ -0,0 +1,232 @@+package onnx++import (+ "errors"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++func goodInputs() []tensorInfo {+ return []tensorInfo{+ {"input_ids", "int64", []int64{-1, -1}}, {"attention_mask", "int64", []int64{-1, -1}},+ {"marker_pos", "int64", []int64{-1, -1}}, {"marker_mask", "bool", []int64{-1, -1}},+ {"qtype", "int64", []int64{-1}},+ }+}++func goodOutputs() []tensorInfo {+ return []tensorInfo{{"logits", "float32", []int64{-1, -1}}, {"act_logits", "float32", []int64{-1, 2}}}+}++func TestCheckGraph(t *testing.T) {+ cases := []struct {+ name string+ mutate func(in, out []tensorInfo) ([]tensorInfo, []tensorInfo)+ problem string // "" for accepted+ }{+ {"current export", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i, o }, ""},+ {"reordered inputs", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0], i[4] = i[4], i[0]; return i, o }, ""},+ {"act_logits fully dynamic", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{-1, -1}; return i, o }, ""},+ {"fixed batch on act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{1, 2}; return i, o }, `"act_logits" declares a fixed batch dimension 1`},+ {"fixed batch on an input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[2].Dims = []int64{2, -1}; return i, o }, `"marker_pos" declares a fixed batch dimension 2`},+ {"fixed batch on qtype", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[4].Dims = []int64{1}; return i, o }, `"qtype" declares a fixed batch`},+ {"fixed sequence length", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, 17}; return i, o }, `"input_ids" declares a fixed size 17`},+ {"fixed marker count", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[0].Dims = []int64{-1, 3}; return i, o }, `"logits" declares a fixed size 3`},+ {"act_logits width other than 2", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{-1, 3}; return i, o }, `"act_logits" declares a fixed size 3`},+ {"input_ids width of 2 is not act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, 2}; return i, o }, `"input_ids" declares a fixed size 2`},+ {"missing input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i[:4], o }, `missing input "qtype"`},+ {"missing output", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i, o[:1] }, `missing output "act_logits"`},+ {"renamed input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[1].Name = "mask"; return i, o }, `missing input "attention_mask"`},+ {"extra input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) {+ return append(i, tensorInfo{"token_type_ids", "int64", []int64{-1, -1}}), o+ }, `unexpected input "token_type_ids"`},+ {"extra output", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) {+ return i, append(o, tensorInfo{"hidden", "float32", []int64{-1, -1}})+ }, `unexpected output "hidden"`},+ {"int32 ids", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].DType = "int32"; return i, o }, `input "input_ids" is int32, want int64`},+ {"int64 marker_mask", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[3].DType = "int64"; return i, o }, `input "marker_mask" is int64, want bool`},+ {"float16 logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[0].DType = "float16"; return i, o }, `output "logits" is float16, want float32`},+ {"rank 3 ids", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, -1, -1}; return i, o }, `input "input_ids" has rank 3, want 2`},+ {"qtype as [batch,1]", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[4].Dims = []int64{-1, 1}; return i, o }, `input "qtype" has rank 2, want 1`},+ {"scalar act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = nil; return i, o }, `output "act_logits" has rank 0, want 2`},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ err := checkGraph(c.mutate(goodInputs(), goodOutputs()))+ if c.problem == "" {+ if err != nil {+ t.Fatalf("checkGraph = %v", err)+ }+ return+ }+ var ge *GraphError+ if !errors.As(err, &ge) || !strings.Contains(ge.Problem, c.problem) {+ t.Fatalf("checkGraph = %v, want a *GraphError containing %q", err, c.problem)+ }+ if !strings.Contains(ge.Problem, "scripts/export_onnx.py") {+ t.Errorf("no re-export hint in %q", ge.Problem)+ }+ })+ }+}++func TestGraphErrorMessage(t *testing.T) {+ err := &GraphError{Path: "a/laya.onnx", Problem: "broken"}+ if got := err.Error(); got != "onnx: graph a/laya.onnx: broken" {+ t.Fatalf("Error = %q", got)+ }+}++func TestCheckPath(t *testing.T) {+ dir := t.TempDir()+ touch := func(parts ...string) string {+ p := filepath.Join(append([]string{dir}, parts...)...)+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {+ t.Fatal(err)+ }+ return p+ }+ fused := touch("fused", "laya.onnx")+ touch("splitdir", "encoder.onnx")+ touch("splitdir", "head.onnx")+ half := touch("half", "encoder.onnx")+ cases := []struct {+ name, path, problem string+ }{+ {"fused file", fused, ""},+ {"half a split pair is only a file", half, ""},+ {"missing file", filepath.Join(dir, "fused", "nope.onnx"), "no such file"},+ {"directory", filepath.Join(dir, "fused"), "is a directory"},+ {"split directory", filepath.Join(dir, "splitdir"), "split encoder.onnx + head.onnx layout is not supported"},+ {"missing laya.onnx beside a split pair", filepath.Join(dir, "splitdir", "laya.onnx"), "no such file; the split"},+ {"the encoder half itself", filepath.Join(dir, "splitdir", "encoder.onnx"), "split encoder.onnx + head.onnx layout"},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ err := checkPath(c.path)+ if c.problem == "" {+ if err != nil {+ t.Fatalf("checkPath = %v", err)+ }+ return+ }+ var ge *GraphError+ if !errors.As(err, &ge) || ge.Path != c.path || !strings.Contains(ge.Problem, c.problem) {+ t.Fatalf("checkPath = %v, want a *GraphError for %s containing %q", err, c.path, c.problem)+ }+ })+ }+}++func TestCollect(t *testing.T) {+ b := infer.Batch{Rows: 2, Length: 4, Markers: 3}+ logits := outTensor{[]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}}+ act := outTensor{[]int64{2, 2}, []float32{7, 8, 9, 10}}+ out, err := collect(b, logits, act)+ if err != nil || len(out.Logits) != 6 || len(out.ActLogits) != 4 || out.Logits[5] != 6 || out.ActLogits[3] != 10 {+ t.Fatalf("collect = %+v, %v", out, err)+ }+ out.Logits[0] = 99+ if logits.data[0] != 1 {+ t.Error("collect returned the runtime's memory, not a copy")+ }++ cases := []struct {+ name string+ logits, act outTensor+ want string+ shapeErrName string+ }{+ {"transposed logits of the same size", outTensor{[]int64{3, 2}, logits.data}, act, "output logits has shape [3 2], want [2 3]", ""},+ {"act_logits in the logits slot", outTensor{[]int64{2, 2}, act.data}, logits, "output logits has shape [2 2], want [2 3]", ""},+ {"logits in the act_logits slot", logits, outTensor{[]int64{2, 3}, logits.data}, "output act_logits has shape [2 3], want [2 2]", ""},+ {"rank 1", outTensor{[]int64{6}, logits.data}, act, "output logits has shape [6]", ""},+ {"short data under the right shape", outTensor{[]int64{2, 3}, logits.data[:5]}, act, "logits has 5 values, want 6", "logits"},+ {"short act data under the right shape", logits, outTensor{[]int64{2, 2}, act.data[:3]}, "act_logits has 3 values, want 4", "act_logits"},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ _, err := collect(b, c.logits, c.act)+ if err == nil || !strings.Contains(err.Error(), c.want) {+ t.Fatalf("collect = %v, want %q", err, c.want)+ }+ var se *infer.ShapeError+ if (c.shapeErrName != "") != errors.As(err, &se) || (se != nil && se.Name != c.shapeErrName) {+ t.Fatalf("collect = %v, want ShapeError for %q", err, c.shapeErrName)+ }+ })+ }+}++func TestLibraryPath(t *testing.T) {+ cases := []struct{ explicit, env, want string }{+ {"/a/lib.so", "/b/lib.so", "/a/lib.so"},+ {"", "/b/lib.so", "/b/lib.so"},+ {"/a/lib.so", "", "/a/lib.so"},+ }+ for _, c := range cases {+ if got := libraryPath(c.explicit, c.env); got != c.want {+ t.Errorf("libraryPath(%q, %q) = %q, want %q", c.explicit, c.env, got, c.want)+ }+ }+ for goos, want := range map[string]string{+ "darwin": "libonnxruntime.dylib", "linux": "libonnxruntime.so", "windows": "onnxruntime.dll", "freebsd": "libonnxruntime.so",+ } {+ if got := defaultLibrary(goos); got != want {+ t.Errorf("defaultLibrary(%s) = %q, want %q", goos, got, want)+ }+ }+ if got := libraryPath("", ""); got != defaultLibrary(runtime.GOOS) {+ t.Errorf("libraryPath(\"\", \"\") = %q", got)+ }+}++func TestLoadErrorHint(t *testing.T) {+ err := loadError("/x/libonnxruntime.so", errors.New("Error setting ORT API base: 2"))+ if !errors.Is(err, ErrRuntime) {+ t.Fatalf("%v does not wrap ErrRuntime", err)+ }+ for _, want := range []string{`"/x/libonnxruntime.so"`, "Error setting ORT API base", "1.29 or newer", LibraryEnv, "WithLibrary"} {+ if !strings.Contains(err.Error(), want) {+ t.Errorf("%q lacks %q", err, want)+ }+ }+}++func TestResolveOptions(t *testing.T) {+ o, err := resolve([]Option{WithLibrary("/l"), nil, WithIntraOpThreads(3), WithInterOpThreads(2)})+ if err != nil || o.library != "/l" || o.intra != 3 || o.inter != 2 {+ t.Fatalf("resolve = %+v, %v", o, err)+ }+ for _, opt := range []Option{WithIntraOpThreads(-1), WithInterOpThreads(-1)} {+ if _, err := resolve([]Option{opt}); err == nil || !strings.HasPrefix(err.Error(), "onnx: ") {+ t.Errorf("negative threads: %v", err)+ }+ }+}++func TestSamePath(t *testing.T) {+ dir := t.TempDir()+ file := filepath.Join(dir, "lib.so")+ if err := os.WriteFile(file, nil, 0o644); err != nil {+ t.Fatal(err)+ }+ link := filepath.Join(dir, "link.so")+ if err := os.Symlink(file, link); err != nil {+ t.Skip("no symlinks here")+ }+ if !samePath(file, link) || !samePath(file, filepath.Join(dir, ".", "lib.so")) {+ t.Error("one file under two spellings was called different")+ }+ if samePath(file, filepath.Join(dir, "other.so")) || samePath("/nonexistent/a.so", "/nonexistent/b.so") {+ t.Error("different files were called the same")+ }+}diff --git a/laya-go/onnx/lifecycle_test.go b/laya-go/onnx/lifecycle_test.gonew file mode 100644index 00000000..a4b4b9e4--- /dev/null+++ b/laya-go/onnx/lifecycle_test.go@@ -0,0 +1,296 @@+//go:build cgo++package onnx++import (+ "context"+ "errors"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"++ "github.com/NandhaKishorM/laya/laya-go/infer"+ ort "github.com/yalue/onnxruntime_go"+)++func mustRun(t *testing.T, s *Session) infer.Output {+ t.Helper()+ b := makeBatch(2, 12, 3)+ out, err := s.Run(context.Background(), b)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }+ if err := out.Check(b); err != nil {+ t.Fatal(err)+ }+ return out+}++func TestOpenCloseReopen(t *testing.T) {+ p := need(t)+ closeShared(t)+ link := filepath.Join(t.TempDir(), filepath.Base(p.library))+ if err := os.Symlink(p.library, link); err != nil {+ t.Fatal(err)+ }+ for i, opts := range [][]Option{nil, nil, {WithLibrary(p.library)}, {WithLibrary(link)}} {+ s, err := Open(p.graph, opts...)+ if err != nil {+ t.Fatalf("open %d: %v", i, err)+ }+ if references() != 1 || !ort.IsInitialized() {+ t.Fatalf("open %d: refs %d, initialized %v", i, references(), ort.IsInitialized())+ }+ if v := s.RuntimeVersion(); v == "" || v != ort.GetVersion() {+ t.Fatalf("RuntimeVersion = %q, runtime says %q", v, ort.GetVersion())+ }+ mustRun(t, s)+ if err := s.Close(); err != nil {+ t.Fatalf("close %d: %v", i, err)+ }+ if references() != 0 || ort.IsInitialized() {+ t.Fatalf("close %d: refs %d, initialized %v; the last Close must unload the runtime", i, references(), ort.IsInitialized())+ }+ }+}++func TestTwoSessionsAtOnce(t *testing.T) {+ p := need(t)+ closeShared(t)+ a, err := Open(p.graph)+ if err != nil {+ t.Fatal(err)+ }+ b, err := Open(p.graph, WithLibrary(p.library), WithIntraOpThreads(2), WithInterOpThreads(1))+ if err != nil {+ t.Fatalf("second session: %v", err)+ }+ if references() != 2 {+ t.Fatalf("refs = %d, want 2", references())+ }+ want := mustRun(t, a)+ got := mustRun(t, b)+ for i := range want.Logits {+ if d := want.Logits[i] - got.Logits[i]; d > 1e-3 || d < -1e-3 {+ t.Fatalf("logit %d differs between sessions: %v vs %v", i, want.Logits[i], got.Logits[i])+ }+ }+ if err := a.Close(); err != nil {+ t.Fatal(err)+ }+ if references() != 1 || !ort.IsInitialized() {+ t.Fatalf("closing one of two: refs %d, initialized %v; the runtime must stay loaded", references(), ort.IsInitialized())+ }+ mustRun(t, b)+ if err := b.Close(); err != nil {+ t.Fatal(err)+ }+ if references() != 0 || ort.IsInitialized() {+ t.Fatalf("after both: refs %d, initialized %v", references(), ort.IsInitialized())+ }+}++func TestConflictingExplicitPath(t *testing.T) {+ p := need(t)+ closeShared(t)+ first, err := Open(p.graph) // loaded through LibraryEnv+ if err != nil {+ t.Fatal(err)+ }+ defer first.Close()+ _, err = Open(p.graph, WithLibrary("/nonexistent/libonnxruntime.dylib"))+ if !errors.Is(err, ErrRuntimeConflict) {+ t.Fatalf("Open with another library = %v, want ErrRuntimeConflict", err)+ }+ if references() != 1 {+ t.Fatalf("a refused Open changed the reference count to %d", references())+ }+ mustRun(t, first)+ same, err := Open(p.graph, WithLibrary(p.library))+ if err != nil {+ t.Fatalf("the same library named explicitly: %v", err)+ }+ if references() != 2 {+ t.Fatalf("refs = %d, want 2", references())+ }+ same.Close()+}++func TestBadLibraryPath(t *testing.T) {+ p := need(t)+ closeShared(t)+ // The environment names a good library; the explicit path must win over it.+ bad := "/nonexistent/libonnxruntime.dylib"+ s, err := Open(p.graph, WithLibrary(bad))+ if !errors.Is(err, ErrRuntime) || s != nil || !strings.Contains(err.Error(), bad) {+ t.Fatalf("Open = %v, %v; want ErrRuntime naming %s", s, err, bad)+ }+ if references() != 0 || ort.IsInitialized() {+ t.Fatalf("a failed Open left refs %d, initialized %v", references(), ort.IsInitialized())+ }+ t.Setenv(LibraryEnv, bad)+ if _, err := Open(p.graph); !errors.Is(err, ErrRuntime) {+ t.Fatalf("Open with a bad %s = %v", LibraryEnv, err)+ }+ // A good explicit path overrides a bad environment, and the process recovers from the failures.+ good, err := Open(p.graph, WithLibrary(p.library))+ if err != nil {+ t.Fatalf("recovery: %v", err)+ }+ mustRun(t, good)+ good.Close()+}++func TestAdoptsAnInitializedRuntime(t *testing.T) {+ p := need(t)+ closeShared(t)+ ort.SetSharedLibraryPath(p.library)+ if err := ort.InitializeEnvironment(); err != nil {+ t.Fatal(err)+ }+ defer func() {+ if ort.IsInitialized() {+ ort.DestroyEnvironment()+ }+ }()+ // The path is ignored: the application's runtime is used.+ s, err := Open(p.graph, WithLibrary("/nonexistent/libonnxruntime.dylib"))+ if err != nil {+ t.Fatalf("Open beside an application-initialized runtime: %v", err)+ }+ mustRun(t, s)+ if err := s.Close(); err != nil {+ t.Fatal(err)+ }+ if !ort.IsInitialized() {+ t.Fatal("Close destroyed a runtime the application owns")+ }+ if references() != 0 {+ t.Fatalf("refs = %d", references())+ }+ if err := ort.DestroyEnvironment(); err != nil {+ t.Fatal(err)+ }+ // With the application's runtime gone, the next Open loads its own again.+ s, err = Open(p.graph)+ if err != nil {+ t.Fatal(err)+ }+ s.Close()+ if ort.IsInitialized() {+ t.Fatal("a runtime we loaded was left behind")+ }+}++func TestRunAfterCloseAndDoubleClose(t *testing.T) {+ p := need(t)+ closeShared(t)+ s, err := Open(p.graph)+ if err != nil {+ t.Fatal(err)+ }+ if err := s.Close(); err != nil {+ t.Fatal(err)+ }+ if err := s.Close(); err != nil {+ t.Fatalf("second Close = %v", err)+ }+ if references() != 0 {+ t.Fatalf("double Close released twice: refs %d", references())+ }+ if _, err := s.Run(context.Background(), makeBatch(1, 8, 2)); !errors.Is(err, infer.ErrClosed) {+ t.Fatalf("Run after Close = %v, want infer.ErrClosed", err)+ }+ if s.RuntimeVersion() == "" {+ t.Error("RuntimeVersion is empty after Close")+ }+ // A second session opened and closed around the first one's double Close must not be affected.+ other, err := Open(p.graph)+ if err != nil {+ t.Fatal(err)+ }+ s.Close()+ if references() != 1 {+ t.Fatalf("a stale Close dropped another session's hold: refs %d", references())+ }+ mustRun(t, other)+ other.Close()+}++func TestCloseWaitsForRunsInFlight(t *testing.T) {+ p := need(t)+ closeShared(t)+ s, err := Open(p.graph)+ if err != nil {+ t.Fatal(err)+ }+ b := makeBatch(24, 256, 4)+ var wg sync.WaitGroup+ var runErr error+ wg.Add(1)+ go func() {+ defer wg.Done()+ _, runErr = s.Run(context.Background(), b)+ }()+ for s.runs.Load() == 0 {+ }+ if err := s.Close(); err != nil {+ t.Fatal(err)+ }+ wg.Wait()+ if runErr != nil {+ t.Fatalf("the in-flight run failed: %v", runErr)+ }+}++func TestOtherGraphs(t *testing.T) {+ p := need(t)+ closeShared(t)++ var ge *GraphError+ full := p.sibling("full-multilingual")+ if s, err := Open(full); !errors.As(err, &ge) || s != nil {+ t.Fatalf("Open(full) = %v, %v; want a *GraphError", s, err)+ }+ if ge.Path != full || !strings.Contains(ge.Problem, `"act_logits"`) || !strings.Contains(ge.Problem, "fixed batch") {+ t.Fatalf("GraphError = %v", ge)+ }+ if references() != 0 || ort.IsInitialized() {+ t.Fatalf("a rejected graph left refs %d, initialized %v", references(), ort.IsInitialized())+ }++ legacy, err := Open(p.sibling("legacy-multilingual"))+ if err != nil {+ t.Fatalf("legacy graph: %v", err)+ }+ b := makeBatch(3, 24, 4)+ out, err := legacy.Run(context.Background(), b)+ if err != nil || out.Check(b) != nil {+ t.Fatalf("legacy Run = %v, %v", out, err)+ }+ legacy.Close()++ splitDir := filepath.Dir(p.sibling("multilingual"))+ if s, err := Open(splitDir); !errors.As(err, &ge) || s != nil || !strings.Contains(ge.Problem, "split encoder.onnx + head.onnx layout is not supported") {+ t.Fatalf("Open(split) = %v, %v", s, err)+ }+ if _, err := Open(filepath.Join(splitDir, "laya.onnx")); !errors.As(err, &ge) || !strings.Contains(ge.Problem, "split") {+ t.Fatalf("Open(split/laya.onnx) = %v", err)+ }+ if _, err := Open(filepath.Join(splitDir, "nope.onnx")); !errors.As(err, &ge) {+ t.Fatalf("Open(missing) = %v", err)+ }+ if references() != 0 {+ t.Fatalf("refs = %d", references())+ }+}++func TestOpenRejectsBadOptions(t *testing.T) {+ p := need(t)+ closeShared(t)+ if _, err := Open(p.graph, WithIntraOpThreads(-1)); err == nil || references() != 0 {+ t.Fatalf("Open = %v, refs %d", err, references())+ }+}diff --git a/laya-go/onnx/nocgo_test.go b/laya-go/onnx/nocgo_test.gonew file mode 100644index 00000000..c69bb049--- /dev/null+++ b/laya-go/onnx/nocgo_test.go@@ -0,0 +1,25 @@+//go:build !cgo++package onnx++import (+ "context"+ "errors"+ "testing"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++func TestOpenWithoutCGO(t *testing.T) {+ s, err := Open("laya.onnx", WithLibrary("/x"))+ if !errors.Is(err, ErrNoCGO) || s != nil {+ t.Fatalf("Open = %v, %v; want ErrNoCGO", s, err)+ }+ var stub Session+ if _, err := stub.Run(context.Background(), infer.Batch{}); !errors.Is(err, ErrNoCGO) {+ t.Errorf("Run = %v", err)+ }+ if stub.Close() != nil || stub.RuntimeVersion() != "" {+ t.Error("the stub's Close and RuntimeVersion are not inert")+ }+}diff --git a/laya-go/onnx/options.go b/laya-go/onnx/options.gonew file mode 100644index 00000000..5c336c9d--- /dev/null+++ b/laya-go/onnx/options.go@@ -0,0 +1,95 @@+package onnx++import (+ "errors"+ "fmt"+ "path/filepath"+ "runtime"+)++// LibraryEnv names the environment variable that holds the path of the ONNX Runtime shared+// library when WithLibrary is not used.+const LibraryEnv = "LAYA_ONNXRUNTIME_LIBRARY"++// ErrNoCGO is returned by Open when the package was built with CGO_ENABLED=0.+var ErrNoCGO = errors.New("onnx: built without cgo")++// ErrRuntime wraps every failure to load or start ONNX Runtime, with the path tried and a hint.+var ErrRuntime = errors.New("onnx: cannot load ONNX Runtime")++// ErrRuntimeConflict is returned when Open names a library other than the one already loaded.+var ErrRuntimeConflict = errors.New("onnx: ONNX Runtime already loaded from another path")++// Option adjusts Open.+type Option func(*options)++type options struct {+ library string+ intra, inter int+}++// WithLibrary sets the ONNX Runtime shared library to load. It overrides LibraryEnv.+func WithLibrary(path string) Option { return func(o *options) { o.library = path } }++// WithIntraOpThreads sets the threads one operator may use. 0 keeps the runtime default.+func WithIntraOpThreads(n int) Option { return func(o *options) { o.intra = n } }++// WithInterOpThreads sets the operators that may run at once. 0 keeps the runtime default.+func WithInterOpThreads(n int) Option { return func(o *options) { o.inter = n } }++// resolve applies the options and rejects values that cannot be honored.+func resolve(opts []Option) (options, error) {+ var o options+ for _, opt := range opts {+ if opt != nil {+ opt(&o)+ }+ }+ if o.intra < 0 || o.inter < 0 {+ return o, fmt.Errorf("onnx: thread counts must not be negative (intra %d, inter %d)", o.intra, o.inter)+ }+ return o, nil+}++// libraryPath picks the shared library: the explicit path, else env, else the platform's name+// for the loader to search for.+func libraryPath(explicit, env string) string {+ if explicit != "" {+ return explicit+ }+ if env != "" {+ return env+ }+ return defaultLibrary(runtime.GOOS)+}++func defaultLibrary(goos string) string {+ switch goos {+ case "darwin":+ return "libonnxruntime.dylib"+ case "windows":+ return "onnxruntime.dll"+ }+ return "libonnxruntime.so"+}++// samePath reports whether a and b name one file.+func samePath(a, b string) bool {+ if a == b {+ return true+ }+ resolve := func(p string) string {+ p = filepath.Clean(p)+ if r, err := filepath.EvalSymlinks(p); err == nil {+ return r+ }+ return p+ }+ return resolve(a) == resolve(b)+}++// loadError wraps a failure to start the runtime from path as ErrRuntime, with a hint.+func loadError(path string, err error) error {+ return fmt.Errorf("%w: %q: %v (needs ONNX Runtime 1.29 or newer; set WithLibrary or %s to its shared library)",+ ErrRuntime, path, err, LibraryEnv)+}diff --git a/laya-go/onnx/predict_test.go b/laya-go/onnx/predict_test.gonew file mode 100644index 00000000..2245469b--- /dev/null+++ b/laya-go/onnx/predict_test.go@@ -0,0 +1,494 @@+//go:build cgo++package onnx++import (+ "context"+ "encoding/json"+ "errors"+ "math"+ "os"+ "path/filepath"+ "reflect"+ "sort"+ "sync"+ "testing"++ "github.com/NandhaKishorM/laya/laya-go/agent"+ "github.com/NandhaKishorM/laya/laya-go/internal/fixtures"+ "github.com/NandhaKishorM/laya/laya-go/schema"+ "github.com/NandhaKishorM/laya/laya-go/sequence"+)++// Comparison rules, from laya-java's PredictionGolden: structure and usage exact, numbers within+// a tolerance because Python computes them on whatever CPU it ran on.+const (+ tolerance = 2e-4 // probabilities, confidences, act_probability, noul+ scoreTolerance = 8e-4 // score, the expectation over the probabilities+ decisiveGap = 1e-3 // the chosen label is compared when the top two differ by more+ maxDiffering = 0.05 // of all compared numbers may differ at all+)++type tally struct {+ t testing.TB+ where string+ numbers, differing int+ worst float64+ labels, ties int+ labelDiffs int+ failures int+}++func (c *tally) failf(format string, args ...any) {+ c.failures+++ if c.failures <= 10 {+ c.t.Errorf(c.where+": "+format, args...)+ }+}++func (c *tally) number(path string, got, want json.Number, tol float64) {+ g, errG := got.Float64()+ w, errW := want.Float64()+ if errG != nil || errW != nil {+ c.failf("%s: %q vs %q", path, got, want)+ return+ }+ c.numbers+++ if got != want {+ c.differing+++ }+ d := math.Abs(g - w)+ c.worst = math.Max(c.worst, d)+ if d > tol {+ c.failf("%s: got %v want %v (off by %.2e, tolerance %.0e)", path, g, w, d, tol)+ }+}++// value compares got to want. Strings, booleans, keys and their order are exact.+func (c *tally) value(path, key string, got, want any) {+ switch w := want.(type) {+ case fixtures.Object:+ g, ok := got.(fixtures.Object)+ if !ok || !reflect.DeepEqual(g.Keys(), w.Keys()) {+ c.failf("%s: keys %v, want %v", path, got, w.Keys())+ return+ }+ for _, m := range w {+ v, _ := g.Get(m.Key)+ c.value(path+"."+m.Key, m.Key, v, m.Value)+ }+ case []any:+ g, ok := got.([]any)+ if !ok || len(g) != len(w) {+ c.failf("%s: %v, want %d items", path, got, len(w))+ return+ }+ for i := range w {+ c.value(path+"[]", key, g[i], w[i])+ }+ case json.Number:+ g, ok := got.(json.Number)+ if !ok {+ c.failf("%s: %v, want a number", path, got)+ return+ }+ tol := tolerance+ if key == "score" {+ tol = scoreTolerance+ }+ c.number(path, g, w, tol)+ default:+ if !reflect.DeepEqual(got, want) {+ c.failf("%s: %v, want %v", path, got, want)+ }+ }+}++func without(o fixtures.Object, key string) fixtures.Object {+ var out fixtures.Object+ for _, m := range o {+ if m.Key != key {+ out = append(out, m)+ }+ }+ return out+}++// answer compares one answer; the chosen label is held to Python's only when it was decisive.+func (c *tally) answer(path string, got, want fixtures.Object) {+ if typ, _ := want.Get("type"); typ == "choice" {+ var ps []float64+ for _, m := range fixtures.Obj(c.t, want.Must("probabilities")) {+ ps = append(ps, fixtures.Float(c.t, m.Value))+ }+ sort.Sort(sort.Reverse(sort.Float64Slice(ps)))+ if len(ps) > 1 && ps[0]-ps[1] <= decisiveGap {+ c.ties+++ c.value(path, "", without(got, "choice"), without(want, "choice"))+ return+ }+ c.labels+++ if got.Must("choice") != want.Must("choice") {+ c.labelDiffs+++ }+ }+ c.value(path, "", got, want)+}++// prediction compares one result: model and usage exactly, answers by the rules above.+func (c *tally) prediction(path string, p schema.Prediction, want fixtures.Object) {+ data, err := json.Marshal(p)+ if err != nil {+ c.t.Fatal(err)+ }+ parsed, err := fixtures.Parse(data)+ if err != nil {+ c.t.Fatal(err)+ }+ got := fixtures.Obj(c.t, parsed)+ c.value(path+".model", "model", got.Must("model"), want.Must("model"))+ if g, w := fixtures.Marshal(c.t, got.Must("usage")), fixtures.Marshal(c.t, want.Must("usage")); string(g) != string(w) {+ c.failf("%s usage\n got %s\nwant %s", path, g, w)+ }+ gotAnswers, wantAnswers := fixtures.Obj(c.t, got.Must("answers")), fixtures.Obj(c.t, want.Must("answers"))+ if !reflect.DeepEqual(gotAnswers.Keys(), wantAnswers.Keys()) {+ c.failf("%s answer ids %v, want %v", path, gotAnswers.Keys(), wantAnswers.Keys())+ return+ }+ for _, m := range wantAnswers {+ g, _ := gotAnswers.Get(m.Key)+ c.answer(path+"."+m.Key, fixtures.Obj(c.t, g), fixtures.Obj(c.t, m.Value))+ }+}++func (c *tally) report(name string) {+ c.t.Logf("%s: %d numbers compared, %d differ at all, worst deviation %.2e, %d labels checked (%d differ), %d near-ties skipped",+ name, c.numbers, c.differing, c.worst, c.labels, c.labelDiffs, c.ties)+}++func (c *tally) finish(name string, minNumbers int) {+ c.report(name)+ if c.numbers < minNumbers {+ c.t.Errorf("%s: only %d numbers were compared", name, c.numbers)+ }+ if float64(c.differing) > maxDiffering*float64(c.numbers) {+ c.t.Errorf("%s: %d of %d numbers differ (more than %.0f%%)", name, c.differing, c.numbers, maxDiffering*100)+ }+}++func openAgent(t testing.TB, p paths, s *Session) *agent.Agent {+ t.Helper()+ a, err := agent.Open(os.DirFS(filepath.Join(p.checkpoints, "multilingual")), keepOpen{s})+ if err != nil {+ t.Fatalf("agent.Open: %v", err)+ }+ t.Cleanup(func() { a.Close() })+ return a+}++type singleCase struct {+ name string+ state any+ lang string+ qs schema.Questions+ want fixtures.Object+}++func loadState(t testing.TB, v any) any {+ t.Helper()+ st, err := sequence.ParseState(fixtures.Marshal(t, v))+ if err != nil {+ t.Fatal(err)+ }+ return st+}++func loadQuestions(t testing.TB, v any) schema.Questions {+ t.Helper()+ qs, err := schema.ParseQuestions(fixtures.Marshal(t, v))+ if err != nil {+ t.Fatal(err)+ }+ return qs+}++func singles(t testing.TB) []singleCase {+ t.Helper()+ golden := fixtures.Load(t, "predict.json")+ var out []singleCase+ for _, m := range fixtures.Obj(t, golden.Must("single")) {+ rec := fixtures.Obj(t, m.Value)+ out = append(out, singleCase{+ name: m.Key, state: loadState(t, rec.Must("state")), lang: fixtures.StrOrEmpty(t, rec.Must("lang")),+ qs: loadQuestions(t, rec.Must("questions")), want: rec,+ })+ }+ return out+}++func (sc singleCase) options() []agent.CallOption {+ if sc.lang == "" {+ return nil+ }+ return []agent.CallOption{agent.WithLanguage(sc.lang)}+}++// TestPredictGolden holds the whole path (agent, collator, this session, ONNX Runtime) to the+// answers Python's ONNXAgent recorded on the same graph.+func TestPredictGolden(t *testing.T) {+ p := need(t)+ a := openAgent(t, p, sharedSession(t))+ golden := fixtures.Load(t, "predict.json")+ if cp := fixtures.Str(t, golden.Must("checkpoint")); cp != "multilingual" {+ t.Fatalf("the golden was recorded on the %s checkpoint", cp)+ }+ ctx := context.Background()++ total := &tally{t: t}+ t.Run("single", func(t *testing.T) {+ cases := singles(t)+ if len(cases) != 6 {+ t.Fatalf("%d single cases", len(cases))+ }+ c := &tally{t: t}+ for _, sc := range cases {+ c.where = sc.name+ pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)+ if err != nil {+ t.Fatalf("%s: %v", sc.name, err)+ }+ c.prediction(sc.name, pred, sc.want)+ }+ c.finish("single", 60)+ merge(total, c)+ })++ t.Run("batch", func(t *testing.T) {+ batches := fixtures.List(t, golden.Must("batch"))+ if len(batches) != 3 {+ t.Fatalf("%d batch configurations", len(batches))+ }+ for _, raw := range batches {+ rec := fixtures.Obj(t, raw)+ var opts []agent.CallOption+ name := "all-in-one"+ if n := rec.Must("batch_size"); n != nil {+ size := fixtures.Int(t, n)+ opts = append(opts, agent.WithBatchSize(size))+ name = "batch-" + string(rune('0'+size))+ }+ if fixtures.Bool(t, rec.Must("sort_by_length")) {+ opts = append(opts, agent.WithSortByLength(true))+ name += "-sorted"+ }+ t.Run(name, func(t *testing.T) {+ var states []any+ for _, s := range fixtures.List(t, rec.Must("states")) {+ states = append(states, loadState(t, s))+ }+ preds, err := a.PredictBatch(ctx, states, loadQuestions(t, rec.Must("questions")), opts...)+ if err != nil {+ t.Fatal(err)+ }+ want := fixtures.List(t, rec.Must("results"))+ if len(preds) != len(want) || len(want) != 9 {+ t.Fatalf("%d results, golden has %d", len(preds), len(want))+ }+ c := &tally{t: t}+ for i, pred := range preds {+ c.where = name+ c.prediction(name+"/state"+string(rune('0'+i)), pred, fixtures.Obj(t, want[i]))+ }+ c.finish(name, 80)+ merge(total, c)+ })+ }+ })+ total.report("all")+ if total.labelDiffs != 0 {+ t.Errorf("%d decisive labels differ from Python", total.labelDiffs)+ }+}++func merge(dst, src *tally) {+ dst.numbers += src.numbers+ dst.differing += src.differing+ dst.worst = math.Max(dst.worst, src.worst)+ dst.labels += src.labels+ dst.labelDiffs += src.labelDiffs+ dst.ties += src.ties+}++// TestConcurrentPredict calls one agent from eight goroutines at once; each answer must equal+// the serial one, and -race watches the Go side.+func TestConcurrentPredict(t *testing.T) {+ p := need(t)+ a := openAgent(t, p, sharedSession(t))+ cases := singles(t)+ ctx := context.Background()+ serial := make([][]byte, len(cases))+ for i, sc := range cases {+ pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)+ if err != nil {+ t.Fatal(err)+ }+ serial[i], _ = json.Marshal(pred)+ }+ var wg sync.WaitGroup+ errs := make(chan error, 8*len(cases))+ for g := 0; g < 8; g++ {+ wg.Add(1)+ go func(g int) {+ defer wg.Done()+ for k := range cases {+ i := (k + g) % len(cases)+ pred, err := a.Predict(ctx, cases[i].state, cases[i].qs, cases[i].options()...)+ if err != nil {+ errs <- err+ continue+ }+ if got, _ := json.Marshal(pred); string(got) != string(serial[i]) {+ errs <- errors.New(cases[i].name + ": concurrent answer differs from the serial one:\n" + string(got) + "\n" + string(serial[i]))+ }+ }+ }(g)+ }+ wg.Wait()+ close(errs)+ for err := range errs {+ t.Error(err)+ }+}++func TestPredictWithCancelledContext(t *testing.T) {+ p := need(t)+ s := sharedSession(t)+ a := openAgent(t, p, s)+ sc := singles(t)[0]+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ before := s.runs.Load()+ if _, err := a.Predict(ctx, sc.state, sc.qs); !errors.Is(err, context.Canceled) {+ t.Fatalf("Predict = %v, want context.Canceled", err)+ }+ if s.runs.Load() != before {+ t.Fatal("the model ran for a cancelled context")+ }+}++func TestRuntimeVersion(t *testing.T) {+ s := sharedSession(t)+ t.Logf("ONNX Runtime %s", s.RuntimeVersion())+ if s.RuntimeVersion() == "" {+ t.Fatal("empty RuntimeVersion")+ }+}++// TestMarkerMaskReachesTheGraph pins that marker_mask is sent as given: padded slots must come+// back different from real ones, so a session that sent all true would be caught.+func TestMarkerMaskReachesTheGraph(t *testing.T) {+ s := sharedSession(t)+ masked := makeBatch(2, 16, 4)+ for r := 0; r < masked.Rows; r++ {+ masked.MarkerMask[r*4+2], masked.MarkerMask[r*4+3] = false, false+ }+ open := makeBatch(2, 16, 4)+ a, err := s.Run(context.Background(), masked)+ if err != nil {+ t.Fatal(err)+ }+ b, err := s.Run(context.Background(), open)+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("masked %v\nopen %v", a.Logits, b.Logits)+ same := true+ for i := range a.Logits {+ same = same && a.Logits[i] == b.Logits[i]+ }+ if same {+ t.Fatal("marker_mask has no effect on the logits")+ }+ if !masked.MarkerMask[0] || masked.MarkerMask[2] {+ t.Fatal("Run modified the batch")+ }+}++// TestTemperaturesReachTheAnswers pins that the checkpoint's temperatures are applied by type and+// by option bucket. The shipped multilingual checkpoint uses 1.0 everywhere, which hides a+// decode that ignores them, so this runs the same session under edited copies of its config.+func TestTemperaturesReachTheAnswers(t *testing.T) {+ p := need(t)+ s := sharedSession(t)+ src := filepath.Join(p.checkpoints, "multilingual")+ raw, err := os.ReadFile(filepath.Join(src, "rl_agent_config.json"))+ if err != nil {+ t.Fatal(err)+ }+ sc := singles(t)[0]+ if sc.name != "billing-mixed" {+ t.Fatalf("first case is %s", sc.name)+ }+ answers := func(edit func(cfg map[string]any)) map[string]string {+ var cfg map[string]any+ if err := json.Unmarshal(raw, &cfg); err != nil {+ t.Fatal(err)+ }+ edit(cfg)+ dir := t.TempDir()+ entries, err := os.ReadDir(src)+ if err != nil {+ t.Fatal(err)+ }+ for _, e := range entries {+ if e.Name() != "rl_agent_config.json" {+ if err := os.Symlink(filepath.Join(src, e.Name()), filepath.Join(dir, e.Name())); err != nil {+ t.Fatal(err)+ }+ }+ }+ data, _ := json.Marshal(cfg)+ if err := os.WriteFile(filepath.Join(dir, "rl_agent_config.json"), data, 0o644); err != nil {+ t.Fatal(err)+ }+ a, err := agent.Open(os.DirFS(dir), keepOpen{s})+ if err != nil {+ t.Fatal(err)+ }+ defer a.Close()+ pred, err := a.Predict(context.Background(), sc.state, sc.qs)+ if err != nil {+ t.Fatal(err)+ }+ data, _ = json.Marshal(pred)+ parsed, err := fixtures.Parse(data)+ if err != nil {+ t.Fatal(err)+ }+ out := map[string]string{}+ for _, m := range fixtures.Obj(t, fixtures.Field(t, parsed, "answers")) {+ out[m.Key] = string(fixtures.Marshal(t, m.Value))+ }+ return out+ }+ base := answers(func(map[string]any) {})+ if len(base) != 3 {+ t.Fatalf("base answers: %v", base)+ }+ ids := map[string]string{"choice": "intent", "score": "urgency", "noul": "duplicate"}+ for i, typ := range []string{"choice", "score", "noul"} {+ temps := []any{1.0, 1.0, 1.0}+ temps[i] = 4.0+ got := answers(func(c map[string]any) { c["temperature"] = temps })+ for _, other := range []string{"choice", "score", "noul"} {+ changed := got[ids[other]] != base[ids[other]]+ if changed != (other == typ) {+ t.Errorf("temperature %v: %s answer changed = %v", temps, other, changed)+ }+ }+ }+ got := answers(func(c map[string]any) { c["temperature_by_options"] = map[string]any{"choice:3-5": 4.0} })+ if got["intent"] == base["intent"] || got["urgency"] != base["urgency"] || got["duplicate"] != base["duplicate"] {+ t.Errorf("a choice:3-5 bucket temperature did not change exactly the 3-option choice: %v", got)+ }+}diff --git a/laya-go/onnx/runtime.go b/laya-go/onnx/runtime.gonew file mode 100644index 00000000..abb163c0--- /dev/null+++ b/laya-go/onnx/runtime.go@@ -0,0 +1,69 @@+//go:build cgo++package onnx++import (+ "fmt"+ "os"+ "sync"++ ort "github.com/yalue/onnxruntime_go"+)++// The binding's environment is process-global and its init and destroy are unsynchronized, so+// this file is the only place that calls them, behind one mutex.+var rt struct {+ mu sync.Mutex+ refs int // sessions holding the environment+ owned bool // we initialized it, so the last release destroys it+ path string // the library we loaded, when owned+ version string+}++// acquire takes a reference on the runtime, starting it on first use, and returns its version.+func acquire(explicit string) (string, error) {+ rt.mu.Lock()+ defer rt.mu.Unlock()+ if rt.refs > 0 {+ if rt.owned && explicit != "" && !samePath(explicit, rt.path) {+ return "", fmt.Errorf("%w: %q is loaded, %q was requested", ErrRuntimeConflict, rt.path, explicit)+ }+ rt.refs+++ return rt.version, nil+ }+ if ort.IsInitialized() {+ rt.owned, rt.path, rt.version = false, "", ort.GetVersion()+ rt.refs = 1+ return rt.version, nil+ }+ path := libraryPath(explicit, os.Getenv(LibraryEnv))+ ort.SetSharedLibraryPath(path)+ if err := ort.InitializeEnvironment(); err != nil {+ return "", loadError(path, err)+ }+ rt.owned, rt.path, rt.version, rt.refs = true, path, ort.GetVersion(), 1+ return rt.version, nil+}++// release drops a reference and unloads the runtime when it was the last and we loaded it.+func release() error {+ rt.mu.Lock()+ defer rt.mu.Unlock()+ rt.refs--+ if rt.refs > 0 {+ return nil+ }+ owned := rt.owned+ rt.owned, rt.path, rt.version = false, "", ""+ if owned {+ return ort.DestroyEnvironment()+ }+ return nil+}++// references reports the sessions holding the runtime, for tests.+func references() int {+ rt.mu.Lock()+ defer rt.mu.Unlock()+ return rt.refs+}diff --git a/laya-go/onnx/session_cgo.go b/laya-go/onnx/session_cgo.gonew file mode 100644index 00000000..8d96aa2c--- /dev/null+++ b/laya-go/onnx/session_cgo.go@@ -0,0 +1,235 @@+//go:build cgo++package onnx++import (+ "context"+ "errors"+ "fmt"+ "sync"+ "sync/atomic"++ "github.com/NandhaKishorM/laya/laya-go/infer"+ ort "github.com/yalue/onnxruntime_go"+)++// Session runs a fused laya graph on ONNX Runtime's CPU provider. It is safe for concurrent use.+type Session struct {+ mu sync.RWMutex // read-held by Run, write-held by Close+ closed bool+ s *ort.DynamicAdvancedSession+ version string+ runs atomic.Int64 // model runs started, for tests+}++var _ infer.Session = (*Session)(nil)++var (+ inputNames = []string{infer.NameInputIDs, infer.NameAttentionMask, infer.NameMarkerPos, infer.NameMarkerMask, infer.NameQType}+ outputNames = []string{infer.NameLogits, infer.NameActLogits}+)++// Open loads a fused laya.onnx, whose .data sidecar must sit beside it, on the CPU provider.+// A graph that is missing, split, or declares other tensors is a *GraphError.+func Open(graphPath string, opts ...Option) (*Session, error) {+ o, err := resolve(opts)+ if err != nil {+ return nil, err+ }+ if err := checkPath(graphPath); err != nil {+ return nil, err+ }+ version, err := acquire(o.library)+ if err != nil {+ return nil, err+ }+ s, err := load(graphPath, o)+ if err != nil {+ return nil, errors.Join(err, release())+ }+ s.version = version+ return s, nil+}++func load(path string, o options) (*Session, error) {+ in, out, err := ort.GetInputOutputInfo(path)+ if err != nil {+ return nil, fmt.Errorf("onnx: cannot read %s: %w", path, err)+ }+ if err := checkGraph(infos(in), infos(out)); err != nil {+ var ge *GraphError+ if errors.As(err, &ge) {+ ge.Path = path+ }+ return nil, err+ }+ so, err := ort.NewSessionOptions()+ if err != nil {+ return nil, fmt.Errorf("onnx: session options: %w", err)+ }+ defer so.Destroy()+ if err := so.SetGraphOptimizationLevel(ort.GraphOptimizationLevelEnableAll); err != nil {+ return nil, fmt.Errorf("onnx: optimization level: %w", err)+ }+ if o.intra > 0 {+ if err := so.SetIntraOpNumThreads(o.intra); err != nil {+ return nil, fmt.Errorf("onnx: intra-op threads: %w", err)+ }+ }+ if o.inter > 0 {+ if err := so.SetInterOpNumThreads(o.inter); err != nil {+ return nil, fmt.Errorf("onnx: inter-op threads: %w", err)+ }+ }+ ds, err := ort.NewDynamicAdvancedSession(path, inputNames, outputNames, so)+ if err != nil {+ return nil, fmt.Errorf("onnx: cannot load %s (its .data sidecar must sit beside it): %w", path, err)+ }+ return &Session{s: ds}, nil+}++func infos(in []ort.InputOutputInfo) []tensorInfo {+ out := make([]tensorInfo, len(in))+ for i, t := range in {+ out[i] = tensorInfo{Name: t.Name, Dims: []int64(t.Dimensions), DType: "non-tensor"}+ if t.OrtValueType != ort.ONNXTypeTensor {+ continue+ }+ switch t.DataType {+ case ort.TensorElementDataTypeInt64:+ out[i].DType = "int64"+ case ort.TensorElementDataTypeBool:+ out[i].DType = "bool"+ case ort.TensorElementDataTypeFloat:+ out[i].DType = "float32"+ default:+ out[i].DType = t.DataType.String()+ }+ }+ return out+}++// RuntimeVersion is the version of the ONNX Runtime library in use, such as "1.30.0".+func (s *Session) RuntimeVersion() string { return s.version }++// Close waits for runs in flight, releases the graph and drops its hold on ONNX Runtime, which+// is unloaded when the last session closes. It is idempotent.+func (s *Session) Close() error {+ s.mu.Lock()+ defer s.mu.Unlock()+ if s.closed {+ return nil+ }+ s.closed = true+ err := s.s.Destroy()+ s.s = nil+ return errors.Join(err, release())+}++// Run runs one batch. A context that ends mid-run terminates the run and Run returns an error+// wrapping ctx.Err(). Inputs are not modified or retained, and the returned slices are the+// caller's.+func (s *Session) Run(ctx context.Context, b infer.Batch) (infer.Output, error) {+ s.mu.RLock()+ defer s.mu.RUnlock()+ if s.closed {+ return infer.Output{}, infer.ErrClosed+ }+ if err := b.Validate(); err != nil {+ return infer.Output{}, err+ }+ if err := ctx.Err(); err != nil {+ return infer.Output{}, fmt.Errorf("onnx: run not started: %w", err)+ }+ if b.Rows == 0 {+ return infer.Output{}, nil+ }++ var vals []ort.Value+ defer func() {+ for _, v := range vals {+ if v != nil {+ v.Destroy()+ }+ }+ }()+ rows, length, markers := int64(b.Rows), int64(b.Length), int64(b.Markers)+ ids, err := ort.NewTensor(ort.NewShape(rows, length), b.InputIDs)+ if err != nil {+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameInputIDs, err)+ }+ vals = append(vals, ids)+ mask, err := ort.NewTensor(ort.NewShape(rows, length), b.AttentionMask)+ if err != nil {+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameAttentionMask, err)+ }+ vals = append(vals, mask)+ pos, err := ort.NewTensor(ort.NewShape(rows, markers), b.MarkerPos)+ if err != nil {+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameMarkerPos, err)+ }+ vals = append(vals, pos)+ mm, err := ort.NewTensor(ort.NewShape(rows, markers), b.MarkerMask)+ if err != nil {+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameMarkerMask, err)+ }+ vals = append(vals, mm)+ qt, err := ort.NewTensor(ort.NewShape(rows), b.QType)+ if err != nil {+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameQType, err)+ }+ vals = append(vals, qt)+ inputs := append([]ort.Value(nil), vals...)++ outputs := []ort.Value{nil, nil}+ defer func() {+ for _, v := range outputs {+ if v != nil {+ v.Destroy()+ }+ }+ }()++ s.runs.Add(1)+ if err := s.execute(ctx, inputs, outputs); err != nil {+ if cerr := ctx.Err(); cerr != nil {+ return infer.Output{}, fmt.Errorf("onnx: run cancelled: %w", cerr)+ }+ return infer.Output{}, fmt.Errorf("onnx: run failed on %d rows of %d tokens: %w", b.Rows, b.Length, err)+ }+ logits, lok := outputs[0].(*ort.Tensor[float32])+ act, aok := outputs[1].(*ort.Tensor[float32])+ if !lok || !aok {+ return infer.Output{}, fmt.Errorf("onnx: outputs are %T and %T, want float32 tensors", outputs[0], outputs[1])+ }+ return collect(b, outTensor{logits.GetShape(), logits.GetData()}, outTensor{act.GetShape(), act.GetData()})+}++// execute runs the graph, terminating it if ctx ends first. The watcher is always joined before+// the run options are destroyed.+func (s *Session) execute(ctx context.Context, inputs, outputs []ort.Value) error {+ done := ctx.Done()+ if done == nil {+ return s.s.Run(inputs, outputs)+ }+ ro, err := ort.NewRunOptions()+ if err != nil {+ return fmt.Errorf("run options: %w", err)+ }+ defer ro.Destroy()+ stop := make(chan struct{})+ joined := make(chan struct{})+ go func() {+ defer close(joined)+ select {+ case <-done:+ ro.Terminate()+ case <-stop:+ }+ }()+ defer func() {+ close(stop)+ <-joined+ }()+ return s.s.RunWithOptions(inputs, outputs, ro)+}diff --git a/laya-go/onnx/session_nocgo.go b/laya-go/onnx/session_nocgo.gonew file mode 100644index 00000000..41b57a9a--- /dev/null+++ b/laya-go/onnx/session_nocgo.go@@ -0,0 +1,28 @@+//go:build !cgo++package onnx++import (+ "context"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++// Session is the ONNX Runtime session. Without cgo it cannot be opened.+type Session struct{}++var _ infer.Session = (*Session)(nil)++// Open returns ErrNoCGO: this build has no ONNX Runtime binding.+func Open(graphPath string, opts ...Option) (*Session, error) { return nil, ErrNoCGO }++// Run returns ErrNoCGO.+func (s *Session) Run(ctx context.Context, b infer.Batch) (infer.Output, error) {+ return infer.Output{}, ErrNoCGO+}++// Close does nothing.+func (s *Session) Close() error { return nil }++// RuntimeVersion returns "".+func (s *Session) RuntimeVersion() string { return "" }diff --git a/laya-go/onnx/support_test.go b/laya-go/onnx/support_test.gonew file mode 100644index 00000000..238f162e--- /dev/null+++ b/laya-go/onnx/support_test.go@@ -0,0 +1,104 @@+//go:build cgo++package onnx++import (+ "os"+ "path/filepath"+ "sync"+ "testing"++ "github.com/NandhaKishorM/laya/laya-go/infer"+)++// paths holds the three locations the model-backed tests need.+type paths struct{ graph, library, checkpoints string }++// need returns the model-backed tests' inputs, skipping when one is unset. CI turns a skip into+// a failure.+func need(t testing.TB) paths {+ t.Helper()+ p := paths{os.Getenv("LAYA_ONNX_GRAPH"), os.Getenv(LibraryEnv), os.Getenv("LAYA_CHECKPOINTS")}+ if p.graph == "" || p.library == "" || p.checkpoints == "" {+ t.Skip("set LAYA_ONNX_GRAPH, LAYA_ONNXRUNTIME_LIBRARY and LAYA_CHECKPOINTS")+ }+ return p+}++// sibling is another export beside the main one: <exports>/<name>/laya.onnx.+func (p paths) sibling(name string) string {+ return filepath.Join(filepath.Dir(filepath.Dir(p.graph)), name, "laya.onnx")+}++// The graph is 1.29 GB, so tests that are not about lifecycle share one session. Lifecycle tests+// call closeShared first, so they start with no runtime loaded.+var shared struct {+ mu sync.Mutex+ s *Session+}++func sharedSession(t testing.TB) *Session {+ t.Helper()+ p := need(t)+ shared.mu.Lock()+ defer shared.mu.Unlock()+ if shared.s == nil {+ s, err := Open(p.graph)+ if err != nil {+ t.Fatalf("Open(%s): %v", p.graph, err)+ }+ shared.s = s+ }+ return shared.s+}++func closeShared(t testing.TB) {+ t.Helper()+ shared.mu.Lock()+ defer shared.mu.Unlock()+ if shared.s != nil {+ if err := shared.s.Close(); err != nil {+ t.Fatalf("closing the shared session: %v", err)+ }+ shared.s = nil+ }+ if n := references(); n != 0 {+ t.Fatalf("%d references to the runtime remain", n)+ }+}++func TestMain(m *testing.M) {+ code := m.Run()+ shared.mu.Lock()+ if shared.s != nil {+ shared.s.Close()+ }+ shared.mu.Unlock()+ os.Exit(code)+}++// keepOpen lets a test hand the shared session to agent.Open without the agent closing it.+type keepOpen struct{ infer.Session }++func (keepOpen) Close() error { return nil }++// makeBatch builds a valid batch of rows x length tokens with the given marker slots.+func makeBatch(rows, length, markers int) infer.Batch {+ b := infer.Batch{+ InputIDs: make([]int64, rows*length), AttentionMask: make([]int64, rows*length),+ MarkerPos: make([]int64, rows*markers), MarkerMask: make([]bool, rows*markers),+ QType: make([]int64, rows), Rows: rows, Length: length, Markers: markers,+ }+ for i := range b.InputIDs {+ b.InputIDs[i] = int64(5 + i%97)+ b.AttentionMask[i] = 1+ }+ for i := range b.MarkerPos {+ b.MarkerPos[i] = int64(1 + i%markers*2%(length-1))+ b.MarkerMask[i] = true+ }+ for i := range b.QType {+ b.QType[i] = int64(i % 3)+ }+ return b+}