Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(go): add laya-go/onnx, a nested module that runs the graph with ONNX Runtime

Adeshina Lasisi39c811532d70
diff --git a/laya-go/README.md b/laya-go/README.md
index 43509b8f..16f97e20 100644
--- a/laya-go/README.md
+++ b/laya-go/README.md
@@ -12,6 +12,7 @@ The foundation (language detection, questions, sequences, the tokenizer) and the
| `sequence` | `build_sequence`, `build_head`, `state_room`, `window_budget` and `window_batch_cap`: `EncodeState`, `Build`, `StateRoom`, `PlanWindows`, `BatchCap`. |
| `infer` | The seam to a model runtime: `Batch` (the graph's five inputs), `Output` (its two), `Session`, and the shape checks `Batch.Validate` and `Output.Check`. |
| `agent` | `ONNXAgent.predict` and `predict_batch` over an `infer.Session`: `Open`, `Predict`, `PredictBatch`, `LoadConfig`, with batch size, sort-by-length and context cancellation between runs. |
+| `onnx` | A nested module (cgo, its own `go.mod`): `onnx.Open` runs a fused `laya.onnx` on ONNX Runtime and implements `infer.Session`, for `agent.Open`. Needs an ONNX Runtime 1.29+ shared library; `CGO_ENABLED=0` builds and `Open` returns `ErrNoCGO`. |
| `tokenizer` | A reader and runner for Hugging Face `tokenizer.json` (BPE): normalizer, pre-tokenizer, added tokens, byte fallback, decoder. |
| `internal/unicodedata` | The character predicates `laya/lang.py` leans on, recorded from CPython. |
| `internal/pyjson` | `json.dumps` and `repr` spelled as CPython spells them. |
diff --git a/laya-go/onnx/cancel_test.go b/laya-go/onnx/cancel_test.go
new file mode 100644
index 00000000..d7287f6a
--- /dev/null
+++ b/laya-go/onnx/cancel_test.go
@@ -0,0 +1,127 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "context"
+ "errors"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+func TestAlreadyCancelledContextDoesNotRun(t *testing.T) {
+ s := sharedSession(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ before := s.runs.Load()
+ _, err := s.Run(ctx, makeBatch(2, 12, 3))
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("Run = %v, want context.Canceled", err)
+ }
+ dctx, dcancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
+ defer dcancel()
+ if _, err := s.Run(dctx, makeBatch(2, 12, 3)); !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("Run with an expired deadline = %v", err)
+ }
+ if n := s.runs.Load() - before; n != 0 {
+ t.Fatalf("the model ran %d times for cancelled contexts", n)
+ }
+}
+
+func TestInvalidBatchIsRefusedBeforeRunning(t *testing.T) {
+ s := sharedSession(t)
+ before := s.runs.Load()
+ b := makeBatch(2, 12, 3)
+ b.QType = b.QType[:1]
+ var se *infer.ShapeError
+ if _, err := s.Run(context.Background(), b); !errors.As(err, &se) || se.Name != infer.NameQType {
+ t.Fatalf("Run = %v, want a ShapeError for qtype", err)
+ }
+ if s.runs.Load() != before {
+ t.Fatal("the model ran on an invalid batch")
+ }
+ out, err := s.Run(context.Background(), infer.Batch{})
+ if err != nil || len(out.Logits) != 0 || s.runs.Load() != before {
+ t.Fatalf("empty batch: %v, %v", out, err)
+ }
+}
+
+func TestCancelMidRun(t *testing.T) {
+ s := sharedSession(t)
+ b := makeBatch(24, 256, 4)
+ start := time.Now()
+ if _, err := s.Run(context.Background(), b); err != nil {
+ t.Fatal(err)
+ }
+ full := time.Since(start)
+ t.Logf("uncancelled run of the large batch: %v", full)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ s.runs.Store(0)
+ go func() {
+ for s.runs.Load() == 0 {
+ runtime.Gosched()
+ }
+ time.Sleep(full / 10)
+ cancel()
+ }()
+ start = time.Now()
+ _, err := s.Run(ctx, b)
+ took := time.Since(start)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("Run = %v after %v, want context.Canceled", err, took)
+ }
+ if took > full/2 {
+ t.Fatalf("a run cancelled at 10%% took %v of %v: the terminate flag did not stop it", took, full)
+ }
+ t.Logf("cancelled run returned after %v", took)
+ // The session is still usable, and a fresh context is not poisoned by the old flag.
+ mustRun(t, s)
+}
+
+func TestNoGoroutineLeak(t *testing.T) {
+ s := sharedSession(t)
+ small := makeBatch(2, 16, 3)
+ mustRun(t, s)
+ settle := func() int {
+ for i := 0; i < 100; i++ {
+ runtime.GC()
+ time.Sleep(10 * time.Millisecond)
+ }
+ return runtime.NumGoroutine()
+ }
+ baseline := settle()
+ for i := 0; i < 50; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if _, err := s.Run(ctx, small); !errors.Is(err, context.Canceled) {
+ t.Fatalf("cancelled run %d: %v", i, err)
+ }
+ }
+ for i := 0; i < 50; i++ {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(i%5)*time.Millisecond)
+ _, err := s.Run(ctx, small)
+ cancel()
+ if err != nil && !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("racing run %d: %v", i, err)
+ }
+ }
+ for i := 0; i < 50; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ if _, err := s.Run(ctx, small); err != nil {
+ t.Fatalf("normal run %d: %v", i, err)
+ }
+ cancel()
+ if _, err := s.Run(context.Background(), small); err != nil {
+ t.Fatalf("background run %d: %v", i, err)
+ }
+ }
+ after := settle()
+ t.Logf("goroutines: baseline %d, after 150 cancellable and 50 plain runs %d", baseline, after)
+ if after > baseline {
+ t.Fatalf("goroutine count rose from %d to %d", baseline, after)
+ }
+}
diff --git a/laya-go/onnx/doc.go b/laya-go/onnx/doc.go
new file mode 100644
index 00000000..d689291c
--- /dev/null
+++ b/laya-go/onnx/doc.go
@@ -0,0 +1,15 @@
+// Package onnx runs a laya graph on ONNX Runtime and implements infer.Session, so the pure-Go
+// agent package can answer questions with it.
+//
+// It is a separate module because it needs cgo and a native ONNX Runtime shared library
+// (1.29 or newer, which the caller supplies). Open takes the fused laya.onnx, whose .data
+// sidecar must sit beside it, and runs it on the CPU provider:
+//
+// sess, err := onnx.Open("laya.onnx", onnx.WithLibrary("/opt/onnxruntime/libonnxruntime.so.1.30.0"))
+// a, err := agent.Open(os.DirFS("checkpoint"), sess)
+//
+// The library is found through WithLibrary, else $LAYA_ONNXRUNTIME_LIBRARY, else the platform's
+// default name on the loader's search path. A process holds one ONNX Runtime: sessions share it,
+// and it is unloaded when the last one closes. Built with CGO_ENABLED=0 the package still
+// compiles, and Open returns ErrNoCGO.
+package onnx
diff --git a/laya-go/onnx/example_test.go b/laya-go/onnx/example_test.go
new file mode 100644
index 00000000..c62a900e
--- /dev/null
+++ b/laya-go/onnx/example_test.go
@@ -0,0 +1,35 @@
+package onnx_test
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/NandhaKishorM/laya/laya-go/agent"
+ "github.com/NandhaKishorM/laya/laya-go/onnx"
+ "github.com/NandhaKishorM/laya/laya-go/schema"
+)
+
+// Open a fused graph, hand it to the agent, and ask one question. The agent closes the session.
+func ExampleOpen() {
+ sess, err := onnx.Open("laya.onnx", onnx.WithLibrary("/opt/onnxruntime/lib/libonnxruntime.so.1.30.0"))
+ if err != nil {
+ log.Fatal(err)
+ }
+ a, err := agent.Open(os.DirFS("checkpoint"), sess)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer a.Close()
+
+ qs, err := schema.ParseQuestions([]byte(`{"urgent": {"type": "noul", "instructions": "Is this urgent?"}}`))
+ if err != nil {
+ log.Fatal(err)
+ }
+ pred, err := a.Predict(context.Background(), "The site is down.", qs)
+ if err != nil {
+ log.Fatal(err)
+ }
+ fmt.Println(pred.Answers)
+}
diff --git a/laya-go/onnx/go.mod b/laya-go/onnx/go.mod
new file mode 100644
index 00000000..a118195d
--- /dev/null
+++ b/laya-go/onnx/go.mod
@@ -0,0 +1,10 @@
+module github.com/NandhaKishorM/laya/laya-go/onnx
+
+go 1.22
+
+require (
+ github.com/NandhaKishorM/laya/laya-go v0.0.0-00010101000000-000000000000
+ github.com/yalue/onnxruntime_go v1.36.0
+)
+
+replace github.com/NandhaKishorM/laya/laya-go => ../
diff --git a/laya-go/onnx/go.sum b/laya-go/onnx/go.sum
new file mode 100644
index 00000000..0b8081bf
--- /dev/null
+++ b/laya-go/onnx/go.sum
@@ -0,0 +1,2 @@
+github.com/yalue/onnxruntime_go v1.36.0 h1:iH1Q++DcsyT9sWtN26KYimESlI5hhXpKaChHDS44oV4=
+github.com/yalue/onnxruntime_go v1.36.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
diff --git a/laya-go/onnx/graph.go b/laya-go/onnx/graph.go
new file mode 100644
index 00000000..4fff6f44
--- /dev/null
+++ b/laya-go/onnx/graph.go
@@ -0,0 +1,164 @@
+package onnx
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+// GraphError reports a graph file laya cannot run: it is missing, is not the fused layout, or
+// declares tensors other than the ones the agent feeds.
+type GraphError struct {
+ // Path is the graph file as passed to Open.
+ Path string
+ // Problem says what is wrong and, where it helps, how to fix it.
+ Problem string
+}
+
+// Error names the file and the problem.
+func (e *GraphError) Error() string { return fmt.Sprintf("onnx: graph %s: %s", e.Path, e.Problem) }
+
+const reexport = "re-export with scripts/export_onnx.py"
+
+// tensorInfo is what checkGraph needs to know about one declared input or output. Dims holds -1
+// for a dynamic dimension.
+type tensorInfo struct {
+ Name string
+ DType string // "int64", "bool", "float32", or whatever else the graph declares
+ Dims []int64
+}
+
+type tensorSpec struct {
+ name, dtype string
+ rank int
+ // fixedOK is the one fixed size a non-batch dimension may have, or 0 for none.
+ fixedOK int64
+}
+
+var (
+ wantInputs = []tensorSpec{
+ {infer.NameInputIDs, "int64", 2, 0}, {infer.NameAttentionMask, "int64", 2, 0},
+ {infer.NameMarkerPos, "int64", 2, 0}, {infer.NameMarkerMask, "bool", 2, 0},
+ {infer.NameQType, "int64", 1, 0},
+ }
+ wantOutputs = []tensorSpec{
+ {infer.NameLogits, "float32", 2, 0}, {infer.NameActLogits, "float32", 2, 2},
+ }
+)
+
+// checkGraph verifies the declared tensors are exactly the five inputs and two outputs, with
+// the dtypes and ranks the agent feeds, and that every dimension that varies per call is
+// dynamic. The returned *GraphError has no Path; Open fills it in.
+func checkGraph(inputs, outputs []tensorInfo) error {
+ if err := checkSide("input", inputs, wantInputs); err != nil {
+ return err
+ }
+ return checkSide("output", outputs, wantOutputs)
+}
+
+func checkSide(kind string, got []tensorInfo, want []tensorSpec) error {
+ byName := make(map[string]tensorInfo, len(got))
+ for _, g := range got {
+ byName[g.Name] = g
+ }
+ for _, w := range want {
+ g, ok := byName[w.name]
+ if !ok {
+ return &GraphError{Problem: fmt.Sprintf("missing %s %q; %s", kind, w.name, reexport)}
+ }
+ if g.DType != w.dtype {
+ return &GraphError{Problem: fmt.Sprintf("%s %q is %s, want %s; %s", kind, w.name, g.DType, w.dtype, reexport)}
+ }
+ if len(g.Dims) != w.rank {
+ return &GraphError{Problem: fmt.Sprintf("%s %q has rank %d, want %d; %s", kind, w.name, len(g.Dims), w.rank, reexport)}
+ }
+ for i, d := range g.Dims {
+ if d == -1 || (i > 0 && d == w.fixedOK && d > 0) {
+ continue
+ }
+ what := "a fixed size"
+ if i == 0 {
+ what = "a fixed batch dimension"
+ }
+ return &GraphError{Problem: fmt.Sprintf("%s %q declares %s %d in shape %v; %s", kind, w.name, what, d, g.Dims, reexport)}
+ }
+ }
+ if len(got) != len(want) {
+ for _, g := range got {
+ known := false
+ for _, w := range want {
+ known = known || w.name == g.Name
+ }
+ if !known {
+ return &GraphError{Problem: fmt.Sprintf("unexpected %s %q; the agent feeds only %d inputs and %d outputs; %s", kind, g.Name, len(wantInputs), len(wantOutputs), reexport)}
+ }
+ }
+ }
+ return nil
+}
+
+// checkPath refuses what is plainly not a fused graph file before any runtime is started.
+func checkPath(path string) error {
+ info, err := os.Stat(path)
+ switch {
+ case err == nil && info.IsDir():
+ if split(path) {
+ return &GraphError{Path: path, Problem: splitProblem}
+ }
+ return &GraphError{Path: path, Problem: "is a directory; pass the path of laya.onnx"}
+ case err == nil:
+ if base := filepath.Base(path); (base == "encoder.onnx" || base == "head.onnx") && split(filepath.Dir(path)) {
+ return &GraphError{Path: path, Problem: splitProblem}
+ }
+ return nil
+ case errors.Is(err, fs.ErrNotExist):
+ if split(filepath.Dir(path)) {
+ return &GraphError{Path: path, Problem: "no such file; " + splitProblem}
+ }
+ return &GraphError{Path: path, Problem: "no such file"}
+ }
+ return &GraphError{Path: path, Problem: err.Error()}
+}
+
+const splitProblem = "the split encoder.onnx + head.onnx layout is not supported; " + reexport + " to write one fused laya.onnx"
+
+// split reports whether dir holds both halves of the split layout.
+func split(dir string) bool {
+ for _, name := range []string{"encoder.onnx", "head.onnx"} {
+ if info, err := os.Stat(filepath.Join(dir, name)); err != nil || info.IsDir() {
+ return false
+ }
+ }
+ return true
+}
+
+// outTensor is one output as the runtime returns it.
+type outTensor struct {
+ shape []int64
+ data []float32
+}
+
+// collect checks both outputs against the batch and copies them into slices the caller owns.
+func collect(b infer.Batch, logits, act outTensor) (infer.Output, error) {
+ for _, c := range []struct {
+ name string
+ t outTensor
+ cols int
+ }{{infer.NameLogits, logits, b.Markers}, {infer.NameActLogits, act, 2}} {
+ if len(c.t.shape) != 2 || c.t.shape[0] != int64(b.Rows) || c.t.shape[1] != int64(c.cols) {
+ return infer.Output{}, fmt.Errorf("onnx: output %s has shape %v, want [%d %d]", c.name, c.t.shape, b.Rows, c.cols)
+ }
+ }
+ o := infer.Output{
+ Logits: append([]float32(nil), logits.data...),
+ ActLogits: append([]float32(nil), act.data...),
+ }
+ if err := o.Check(b); err != nil {
+ return infer.Output{}, err
+ }
+ return o, nil
+}
diff --git a/laya-go/onnx/graph_test.go b/laya-go/onnx/graph_test.go
new file mode 100644
index 00000000..3aa7e5ff
--- /dev/null
+++ b/laya-go/onnx/graph_test.go
@@ -0,0 +1,232 @@
+package onnx
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+func goodInputs() []tensorInfo {
+ return []tensorInfo{
+ {"input_ids", "int64", []int64{-1, -1}}, {"attention_mask", "int64", []int64{-1, -1}},
+ {"marker_pos", "int64", []int64{-1, -1}}, {"marker_mask", "bool", []int64{-1, -1}},
+ {"qtype", "int64", []int64{-1}},
+ }
+}
+
+func goodOutputs() []tensorInfo {
+ return []tensorInfo{{"logits", "float32", []int64{-1, -1}}, {"act_logits", "float32", []int64{-1, 2}}}
+}
+
+func TestCheckGraph(t *testing.T) {
+ cases := []struct {
+ name string
+ mutate func(in, out []tensorInfo) ([]tensorInfo, []tensorInfo)
+ problem string // "" for accepted
+ }{
+ {"current export", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i, o }, ""},
+ {"reordered inputs", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0], i[4] = i[4], i[0]; return i, o }, ""},
+ {"act_logits fully dynamic", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{-1, -1}; return i, o }, ""},
+ {"fixed batch on act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{1, 2}; return i, o }, `"act_logits" declares a fixed batch dimension 1`},
+ {"fixed batch on an input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[2].Dims = []int64{2, -1}; return i, o }, `"marker_pos" declares a fixed batch dimension 2`},
+ {"fixed batch on qtype", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[4].Dims = []int64{1}; return i, o }, `"qtype" declares a fixed batch`},
+ {"fixed sequence length", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, 17}; return i, o }, `"input_ids" declares a fixed size 17`},
+ {"fixed marker count", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[0].Dims = []int64{-1, 3}; return i, o }, `"logits" declares a fixed size 3`},
+ {"act_logits width other than 2", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = []int64{-1, 3}; return i, o }, `"act_logits" declares a fixed size 3`},
+ {"input_ids width of 2 is not act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, 2}; return i, o }, `"input_ids" declares a fixed size 2`},
+ {"missing input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i[:4], o }, `missing input "qtype"`},
+ {"missing output", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { return i, o[:1] }, `missing output "act_logits"`},
+ {"renamed input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[1].Name = "mask"; return i, o }, `missing input "attention_mask"`},
+ {"extra input", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) {
+ return append(i, tensorInfo{"token_type_ids", "int64", []int64{-1, -1}}), o
+ }, `unexpected input "token_type_ids"`},
+ {"extra output", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) {
+ return i, append(o, tensorInfo{"hidden", "float32", []int64{-1, -1}})
+ }, `unexpected output "hidden"`},
+ {"int32 ids", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].DType = "int32"; return i, o }, `input "input_ids" is int32, want int64`},
+ {"int64 marker_mask", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[3].DType = "int64"; return i, o }, `input "marker_mask" is int64, want bool`},
+ {"float16 logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[0].DType = "float16"; return i, o }, `output "logits" is float16, want float32`},
+ {"rank 3 ids", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[0].Dims = []int64{-1, -1, -1}; return i, o }, `input "input_ids" has rank 3, want 2`},
+ {"qtype as [batch,1]", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { i[4].Dims = []int64{-1, 1}; return i, o }, `input "qtype" has rank 2, want 1`},
+ {"scalar act_logits", func(i, o []tensorInfo) ([]tensorInfo, []tensorInfo) { o[1].Dims = nil; return i, o }, `output "act_logits" has rank 0, want 2`},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ err := checkGraph(c.mutate(goodInputs(), goodOutputs()))
+ if c.problem == "" {
+ if err != nil {
+ t.Fatalf("checkGraph = %v", err)
+ }
+ return
+ }
+ var ge *GraphError
+ if !errors.As(err, &ge) || !strings.Contains(ge.Problem, c.problem) {
+ t.Fatalf("checkGraph = %v, want a *GraphError containing %q", err, c.problem)
+ }
+ if !strings.Contains(ge.Problem, "scripts/export_onnx.py") {
+ t.Errorf("no re-export hint in %q", ge.Problem)
+ }
+ })
+ }
+}
+
+func TestGraphErrorMessage(t *testing.T) {
+ err := &GraphError{Path: "a/laya.onnx", Problem: "broken"}
+ if got := err.Error(); got != "onnx: graph a/laya.onnx: broken" {
+ t.Fatalf("Error = %q", got)
+ }
+}
+
+func TestCheckPath(t *testing.T) {
+ dir := t.TempDir()
+ touch := func(parts ...string) string {
+ p := filepath.Join(append([]string{dir}, parts...)...)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return p
+ }
+ fused := touch("fused", "laya.onnx")
+ touch("splitdir", "encoder.onnx")
+ touch("splitdir", "head.onnx")
+ half := touch("half", "encoder.onnx")
+ cases := []struct {
+ name, path, problem string
+ }{
+ {"fused file", fused, ""},
+ {"half a split pair is only a file", half, ""},
+ {"missing file", filepath.Join(dir, "fused", "nope.onnx"), "no such file"},
+ {"directory", filepath.Join(dir, "fused"), "is a directory"},
+ {"split directory", filepath.Join(dir, "splitdir"), "split encoder.onnx + head.onnx layout is not supported"},
+ {"missing laya.onnx beside a split pair", filepath.Join(dir, "splitdir", "laya.onnx"), "no such file; the split"},
+ {"the encoder half itself", filepath.Join(dir, "splitdir", "encoder.onnx"), "split encoder.onnx + head.onnx layout"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ err := checkPath(c.path)
+ if c.problem == "" {
+ if err != nil {
+ t.Fatalf("checkPath = %v", err)
+ }
+ return
+ }
+ var ge *GraphError
+ if !errors.As(err, &ge) || ge.Path != c.path || !strings.Contains(ge.Problem, c.problem) {
+ t.Fatalf("checkPath = %v, want a *GraphError for %s containing %q", err, c.path, c.problem)
+ }
+ })
+ }
+}
+
+func TestCollect(t *testing.T) {
+ b := infer.Batch{Rows: 2, Length: 4, Markers: 3}
+ logits := outTensor{[]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}}
+ act := outTensor{[]int64{2, 2}, []float32{7, 8, 9, 10}}
+ out, err := collect(b, logits, act)
+ if err != nil || len(out.Logits) != 6 || len(out.ActLogits) != 4 || out.Logits[5] != 6 || out.ActLogits[3] != 10 {
+ t.Fatalf("collect = %+v, %v", out, err)
+ }
+ out.Logits[0] = 99
+ if logits.data[0] != 1 {
+ t.Error("collect returned the runtime's memory, not a copy")
+ }
+
+ cases := []struct {
+ name string
+ logits, act outTensor
+ want string
+ shapeErrName string
+ }{
+ {"transposed logits of the same size", outTensor{[]int64{3, 2}, logits.data}, act, "output logits has shape [3 2], want [2 3]", ""},
+ {"act_logits in the logits slot", outTensor{[]int64{2, 2}, act.data}, logits, "output logits has shape [2 2], want [2 3]", ""},
+ {"logits in the act_logits slot", logits, outTensor{[]int64{2, 3}, logits.data}, "output act_logits has shape [2 3], want [2 2]", ""},
+ {"rank 1", outTensor{[]int64{6}, logits.data}, act, "output logits has shape [6]", ""},
+ {"short data under the right shape", outTensor{[]int64{2, 3}, logits.data[:5]}, act, "logits has 5 values, want 6", "logits"},
+ {"short act data under the right shape", logits, outTensor{[]int64{2, 2}, act.data[:3]}, "act_logits has 3 values, want 4", "act_logits"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ _, err := collect(b, c.logits, c.act)
+ if err == nil || !strings.Contains(err.Error(), c.want) {
+ t.Fatalf("collect = %v, want %q", err, c.want)
+ }
+ var se *infer.ShapeError
+ if (c.shapeErrName != "") != errors.As(err, &se) || (se != nil && se.Name != c.shapeErrName) {
+ t.Fatalf("collect = %v, want ShapeError for %q", err, c.shapeErrName)
+ }
+ })
+ }
+}
+
+func TestLibraryPath(t *testing.T) {
+ cases := []struct{ explicit, env, want string }{
+ {"/a/lib.so", "/b/lib.so", "/a/lib.so"},
+ {"", "/b/lib.so", "/b/lib.so"},
+ {"/a/lib.so", "", "/a/lib.so"},
+ }
+ for _, c := range cases {
+ if got := libraryPath(c.explicit, c.env); got != c.want {
+ t.Errorf("libraryPath(%q, %q) = %q, want %q", c.explicit, c.env, got, c.want)
+ }
+ }
+ for goos, want := range map[string]string{
+ "darwin": "libonnxruntime.dylib", "linux": "libonnxruntime.so", "windows": "onnxruntime.dll", "freebsd": "libonnxruntime.so",
+ } {
+ if got := defaultLibrary(goos); got != want {
+ t.Errorf("defaultLibrary(%s) = %q, want %q", goos, got, want)
+ }
+ }
+ if got := libraryPath("", ""); got != defaultLibrary(runtime.GOOS) {
+ t.Errorf("libraryPath(\"\", \"\") = %q", got)
+ }
+}
+
+func TestLoadErrorHint(t *testing.T) {
+ err := loadError("/x/libonnxruntime.so", errors.New("Error setting ORT API base: 2"))
+ if !errors.Is(err, ErrRuntime) {
+ t.Fatalf("%v does not wrap ErrRuntime", err)
+ }
+ for _, want := range []string{`"/x/libonnxruntime.so"`, "Error setting ORT API base", "1.29 or newer", LibraryEnv, "WithLibrary"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("%q lacks %q", err, want)
+ }
+ }
+}
+
+func TestResolveOptions(t *testing.T) {
+ o, err := resolve([]Option{WithLibrary("/l"), nil, WithIntraOpThreads(3), WithInterOpThreads(2)})
+ if err != nil || o.library != "/l" || o.intra != 3 || o.inter != 2 {
+ t.Fatalf("resolve = %+v, %v", o, err)
+ }
+ for _, opt := range []Option{WithIntraOpThreads(-1), WithInterOpThreads(-1)} {
+ if _, err := resolve([]Option{opt}); err == nil || !strings.HasPrefix(err.Error(), "onnx: ") {
+ t.Errorf("negative threads: %v", err)
+ }
+ }
+}
+
+func TestSamePath(t *testing.T) {
+ dir := t.TempDir()
+ file := filepath.Join(dir, "lib.so")
+ if err := os.WriteFile(file, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ link := filepath.Join(dir, "link.so")
+ if err := os.Symlink(file, link); err != nil {
+ t.Skip("no symlinks here")
+ }
+ if !samePath(file, link) || !samePath(file, filepath.Join(dir, ".", "lib.so")) {
+ t.Error("one file under two spellings was called different")
+ }
+ if samePath(file, filepath.Join(dir, "other.so")) || samePath("/nonexistent/a.so", "/nonexistent/b.so") {
+ t.Error("different files were called the same")
+ }
+}
diff --git a/laya-go/onnx/lifecycle_test.go b/laya-go/onnx/lifecycle_test.go
new file mode 100644
index 00000000..a4b4b9e4
--- /dev/null
+++ b/laya-go/onnx/lifecycle_test.go
@@ -0,0 +1,296 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+ ort "github.com/yalue/onnxruntime_go"
+)
+
+func mustRun(t *testing.T, s *Session) infer.Output {
+ t.Helper()
+ b := makeBatch(2, 12, 3)
+ out, err := s.Run(context.Background(), b)
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+ if err := out.Check(b); err != nil {
+ t.Fatal(err)
+ }
+ return out
+}
+
+func TestOpenCloseReopen(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ link := filepath.Join(t.TempDir(), filepath.Base(p.library))
+ if err := os.Symlink(p.library, link); err != nil {
+ t.Fatal(err)
+ }
+ for i, opts := range [][]Option{nil, nil, {WithLibrary(p.library)}, {WithLibrary(link)}} {
+ s, err := Open(p.graph, opts...)
+ if err != nil {
+ t.Fatalf("open %d: %v", i, err)
+ }
+ if references() != 1 || !ort.IsInitialized() {
+ t.Fatalf("open %d: refs %d, initialized %v", i, references(), ort.IsInitialized())
+ }
+ if v := s.RuntimeVersion(); v == "" || v != ort.GetVersion() {
+ t.Fatalf("RuntimeVersion = %q, runtime says %q", v, ort.GetVersion())
+ }
+ mustRun(t, s)
+ if err := s.Close(); err != nil {
+ t.Fatalf("close %d: %v", i, err)
+ }
+ if references() != 0 || ort.IsInitialized() {
+ t.Fatalf("close %d: refs %d, initialized %v; the last Close must unload the runtime", i, references(), ort.IsInitialized())
+ }
+ }
+}
+
+func TestTwoSessionsAtOnce(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ a, err := Open(p.graph)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := Open(p.graph, WithLibrary(p.library), WithIntraOpThreads(2), WithInterOpThreads(1))
+ if err != nil {
+ t.Fatalf("second session: %v", err)
+ }
+ if references() != 2 {
+ t.Fatalf("refs = %d, want 2", references())
+ }
+ want := mustRun(t, a)
+ got := mustRun(t, b)
+ for i := range want.Logits {
+ if d := want.Logits[i] - got.Logits[i]; d > 1e-3 || d < -1e-3 {
+ t.Fatalf("logit %d differs between sessions: %v vs %v", i, want.Logits[i], got.Logits[i])
+ }
+ }
+ if err := a.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if references() != 1 || !ort.IsInitialized() {
+ t.Fatalf("closing one of two: refs %d, initialized %v; the runtime must stay loaded", references(), ort.IsInitialized())
+ }
+ mustRun(t, b)
+ if err := b.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if references() != 0 || ort.IsInitialized() {
+ t.Fatalf("after both: refs %d, initialized %v", references(), ort.IsInitialized())
+ }
+}
+
+func TestConflictingExplicitPath(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ first, err := Open(p.graph) // loaded through LibraryEnv
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer first.Close()
+ _, err = Open(p.graph, WithLibrary("/nonexistent/libonnxruntime.dylib"))
+ if !errors.Is(err, ErrRuntimeConflict) {
+ t.Fatalf("Open with another library = %v, want ErrRuntimeConflict", err)
+ }
+ if references() != 1 {
+ t.Fatalf("a refused Open changed the reference count to %d", references())
+ }
+ mustRun(t, first)
+ same, err := Open(p.graph, WithLibrary(p.library))
+ if err != nil {
+ t.Fatalf("the same library named explicitly: %v", err)
+ }
+ if references() != 2 {
+ t.Fatalf("refs = %d, want 2", references())
+ }
+ same.Close()
+}
+
+func TestBadLibraryPath(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ // The environment names a good library; the explicit path must win over it.
+ bad := "/nonexistent/libonnxruntime.dylib"
+ s, err := Open(p.graph, WithLibrary(bad))
+ if !errors.Is(err, ErrRuntime) || s != nil || !strings.Contains(err.Error(), bad) {
+ t.Fatalf("Open = %v, %v; want ErrRuntime naming %s", s, err, bad)
+ }
+ if references() != 0 || ort.IsInitialized() {
+ t.Fatalf("a failed Open left refs %d, initialized %v", references(), ort.IsInitialized())
+ }
+ t.Setenv(LibraryEnv, bad)
+ if _, err := Open(p.graph); !errors.Is(err, ErrRuntime) {
+ t.Fatalf("Open with a bad %s = %v", LibraryEnv, err)
+ }
+ // A good explicit path overrides a bad environment, and the process recovers from the failures.
+ good, err := Open(p.graph, WithLibrary(p.library))
+ if err != nil {
+ t.Fatalf("recovery: %v", err)
+ }
+ mustRun(t, good)
+ good.Close()
+}
+
+func TestAdoptsAnInitializedRuntime(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ ort.SetSharedLibraryPath(p.library)
+ if err := ort.InitializeEnvironment(); err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ if ort.IsInitialized() {
+ ort.DestroyEnvironment()
+ }
+ }()
+ // The path is ignored: the application's runtime is used.
+ s, err := Open(p.graph, WithLibrary("/nonexistent/libonnxruntime.dylib"))
+ if err != nil {
+ t.Fatalf("Open beside an application-initialized runtime: %v", err)
+ }
+ mustRun(t, s)
+ if err := s.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if !ort.IsInitialized() {
+ t.Fatal("Close destroyed a runtime the application owns")
+ }
+ if references() != 0 {
+ t.Fatalf("refs = %d", references())
+ }
+ if err := ort.DestroyEnvironment(); err != nil {
+ t.Fatal(err)
+ }
+ // With the application's runtime gone, the next Open loads its own again.
+ s, err = Open(p.graph)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.Close()
+ if ort.IsInitialized() {
+ t.Fatal("a runtime we loaded was left behind")
+ }
+}
+
+func TestRunAfterCloseAndDoubleClose(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ s, err := Open(p.graph)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.Close(); err != nil {
+ t.Fatalf("second Close = %v", err)
+ }
+ if references() != 0 {
+ t.Fatalf("double Close released twice: refs %d", references())
+ }
+ if _, err := s.Run(context.Background(), makeBatch(1, 8, 2)); !errors.Is(err, infer.ErrClosed) {
+ t.Fatalf("Run after Close = %v, want infer.ErrClosed", err)
+ }
+ if s.RuntimeVersion() == "" {
+ t.Error("RuntimeVersion is empty after Close")
+ }
+ // A second session opened and closed around the first one's double Close must not be affected.
+ other, err := Open(p.graph)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.Close()
+ if references() != 1 {
+ t.Fatalf("a stale Close dropped another session's hold: refs %d", references())
+ }
+ mustRun(t, other)
+ other.Close()
+}
+
+func TestCloseWaitsForRunsInFlight(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ s, err := Open(p.graph)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := makeBatch(24, 256, 4)
+ var wg sync.WaitGroup
+ var runErr error
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ _, runErr = s.Run(context.Background(), b)
+ }()
+ for s.runs.Load() == 0 {
+ }
+ if err := s.Close(); err != nil {
+ t.Fatal(err)
+ }
+ wg.Wait()
+ if runErr != nil {
+ t.Fatalf("the in-flight run failed: %v", runErr)
+ }
+}
+
+func TestOtherGraphs(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+
+ var ge *GraphError
+ full := p.sibling("full-multilingual")
+ if s, err := Open(full); !errors.As(err, &ge) || s != nil {
+ t.Fatalf("Open(full) = %v, %v; want a *GraphError", s, err)
+ }
+ if ge.Path != full || !strings.Contains(ge.Problem, `"act_logits"`) || !strings.Contains(ge.Problem, "fixed batch") {
+ t.Fatalf("GraphError = %v", ge)
+ }
+ if references() != 0 || ort.IsInitialized() {
+ t.Fatalf("a rejected graph left refs %d, initialized %v", references(), ort.IsInitialized())
+ }
+
+ legacy, err := Open(p.sibling("legacy-multilingual"))
+ if err != nil {
+ t.Fatalf("legacy graph: %v", err)
+ }
+ b := makeBatch(3, 24, 4)
+ out, err := legacy.Run(context.Background(), b)
+ if err != nil || out.Check(b) != nil {
+ t.Fatalf("legacy Run = %v, %v", out, err)
+ }
+ legacy.Close()
+
+ splitDir := filepath.Dir(p.sibling("multilingual"))
+ if s, err := Open(splitDir); !errors.As(err, &ge) || s != nil || !strings.Contains(ge.Problem, "split encoder.onnx + head.onnx layout is not supported") {
+ t.Fatalf("Open(split) = %v, %v", s, err)
+ }
+ if _, err := Open(filepath.Join(splitDir, "laya.onnx")); !errors.As(err, &ge) || !strings.Contains(ge.Problem, "split") {
+ t.Fatalf("Open(split/laya.onnx) = %v", err)
+ }
+ if _, err := Open(filepath.Join(splitDir, "nope.onnx")); !errors.As(err, &ge) {
+ t.Fatalf("Open(missing) = %v", err)
+ }
+ if references() != 0 {
+ t.Fatalf("refs = %d", references())
+ }
+}
+
+func TestOpenRejectsBadOptions(t *testing.T) {
+ p := need(t)
+ closeShared(t)
+ if _, err := Open(p.graph, WithIntraOpThreads(-1)); err == nil || references() != 0 {
+ t.Fatalf("Open = %v, refs %d", err, references())
+ }
+}
diff --git a/laya-go/onnx/nocgo_test.go b/laya-go/onnx/nocgo_test.go
new file mode 100644
index 00000000..c69bb049
--- /dev/null
+++ b/laya-go/onnx/nocgo_test.go
@@ -0,0 +1,25 @@
+//go:build !cgo
+
+package onnx
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+func TestOpenWithoutCGO(t *testing.T) {
+ s, err := Open("laya.onnx", WithLibrary("/x"))
+ if !errors.Is(err, ErrNoCGO) || s != nil {
+ t.Fatalf("Open = %v, %v; want ErrNoCGO", s, err)
+ }
+ var stub Session
+ if _, err := stub.Run(context.Background(), infer.Batch{}); !errors.Is(err, ErrNoCGO) {
+ t.Errorf("Run = %v", err)
+ }
+ if stub.Close() != nil || stub.RuntimeVersion() != "" {
+ t.Error("the stub's Close and RuntimeVersion are not inert")
+ }
+}
diff --git a/laya-go/onnx/options.go b/laya-go/onnx/options.go
new file mode 100644
index 00000000..5c336c9d
--- /dev/null
+++ b/laya-go/onnx/options.go
@@ -0,0 +1,95 @@
+package onnx
+
+import (
+ "errors"
+ "fmt"
+ "path/filepath"
+ "runtime"
+)
+
+// LibraryEnv names the environment variable that holds the path of the ONNX Runtime shared
+// library when WithLibrary is not used.
+const LibraryEnv = "LAYA_ONNXRUNTIME_LIBRARY"
+
+// ErrNoCGO is returned by Open when the package was built with CGO_ENABLED=0.
+var ErrNoCGO = errors.New("onnx: built without cgo")
+
+// ErrRuntime wraps every failure to load or start ONNX Runtime, with the path tried and a hint.
+var ErrRuntime = errors.New("onnx: cannot load ONNX Runtime")
+
+// ErrRuntimeConflict is returned when Open names a library other than the one already loaded.
+var ErrRuntimeConflict = errors.New("onnx: ONNX Runtime already loaded from another path")
+
+// Option adjusts Open.
+type Option func(*options)
+
+type options struct {
+ library string
+ intra, inter int
+}
+
+// WithLibrary sets the ONNX Runtime shared library to load. It overrides LibraryEnv.
+func WithLibrary(path string) Option { return func(o *options) { o.library = path } }
+
+// WithIntraOpThreads sets the threads one operator may use. 0 keeps the runtime default.
+func WithIntraOpThreads(n int) Option { return func(o *options) { o.intra = n } }
+
+// WithInterOpThreads sets the operators that may run at once. 0 keeps the runtime default.
+func WithInterOpThreads(n int) Option { return func(o *options) { o.inter = n } }
+
+// resolve applies the options and rejects values that cannot be honored.
+func resolve(opts []Option) (options, error) {
+ var o options
+ for _, opt := range opts {
+ if opt != nil {
+ opt(&o)
+ }
+ }
+ if o.intra < 0 || o.inter < 0 {
+ return o, fmt.Errorf("onnx: thread counts must not be negative (intra %d, inter %d)", o.intra, o.inter)
+ }
+ return o, nil
+}
+
+// libraryPath picks the shared library: the explicit path, else env, else the platform's name
+// for the loader to search for.
+func libraryPath(explicit, env string) string {
+ if explicit != "" {
+ return explicit
+ }
+ if env != "" {
+ return env
+ }
+ return defaultLibrary(runtime.GOOS)
+}
+
+func defaultLibrary(goos string) string {
+ switch goos {
+ case "darwin":
+ return "libonnxruntime.dylib"
+ case "windows":
+ return "onnxruntime.dll"
+ }
+ return "libonnxruntime.so"
+}
+
+// samePath reports whether a and b name one file.
+func samePath(a, b string) bool {
+ if a == b {
+ return true
+ }
+ resolve := func(p string) string {
+ p = filepath.Clean(p)
+ if r, err := filepath.EvalSymlinks(p); err == nil {
+ return r
+ }
+ return p
+ }
+ return resolve(a) == resolve(b)
+}
+
+// loadError wraps a failure to start the runtime from path as ErrRuntime, with a hint.
+func loadError(path string, err error) error {
+ return fmt.Errorf("%w: %q: %v (needs ONNX Runtime 1.29 or newer; set WithLibrary or %s to its shared library)",
+ ErrRuntime, path, err, LibraryEnv)
+}
diff --git a/laya-go/onnx/predict_test.go b/laya-go/onnx/predict_test.go
new file mode 100644
index 00000000..2245469b
--- /dev/null
+++ b/laya-go/onnx/predict_test.go
@@ -0,0 +1,494 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "math"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "sync"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/agent"
+ "github.com/NandhaKishorM/laya/laya-go/internal/fixtures"
+ "github.com/NandhaKishorM/laya/laya-go/schema"
+ "github.com/NandhaKishorM/laya/laya-go/sequence"
+)
+
+// Comparison rules, from laya-java's PredictionGolden: structure and usage exact, numbers within
+// a tolerance because Python computes them on whatever CPU it ran on.
+const (
+ tolerance = 2e-4 // probabilities, confidences, act_probability, noul
+ scoreTolerance = 8e-4 // score, the expectation over the probabilities
+ decisiveGap = 1e-3 // the chosen label is compared when the top two differ by more
+ maxDiffering = 0.05 // of all compared numbers may differ at all
+)
+
+type tally struct {
+ t testing.TB
+ where string
+ numbers, differing int
+ worst float64
+ labels, ties int
+ labelDiffs int
+ failures int
+}
+
+func (c *tally) failf(format string, args ...any) {
+ c.failures++
+ if c.failures <= 10 {
+ c.t.Errorf(c.where+": "+format, args...)
+ }
+}
+
+func (c *tally) number(path string, got, want json.Number, tol float64) {
+ g, errG := got.Float64()
+ w, errW := want.Float64()
+ if errG != nil || errW != nil {
+ c.failf("%s: %q vs %q", path, got, want)
+ return
+ }
+ c.numbers++
+ if got != want {
+ c.differing++
+ }
+ d := math.Abs(g - w)
+ c.worst = math.Max(c.worst, d)
+ if d > tol {
+ c.failf("%s: got %v want %v (off by %.2e, tolerance %.0e)", path, g, w, d, tol)
+ }
+}
+
+// value compares got to want. Strings, booleans, keys and their order are exact.
+func (c *tally) value(path, key string, got, want any) {
+ switch w := want.(type) {
+ case fixtures.Object:
+ g, ok := got.(fixtures.Object)
+ if !ok || !reflect.DeepEqual(g.Keys(), w.Keys()) {
+ c.failf("%s: keys %v, want %v", path, got, w.Keys())
+ return
+ }
+ for _, m := range w {
+ v, _ := g.Get(m.Key)
+ c.value(path+"."+m.Key, m.Key, v, m.Value)
+ }
+ case []any:
+ g, ok := got.([]any)
+ if !ok || len(g) != len(w) {
+ c.failf("%s: %v, want %d items", path, got, len(w))
+ return
+ }
+ for i := range w {
+ c.value(path+"[]", key, g[i], w[i])
+ }
+ case json.Number:
+ g, ok := got.(json.Number)
+ if !ok {
+ c.failf("%s: %v, want a number", path, got)
+ return
+ }
+ tol := tolerance
+ if key == "score" {
+ tol = scoreTolerance
+ }
+ c.number(path, g, w, tol)
+ default:
+ if !reflect.DeepEqual(got, want) {
+ c.failf("%s: %v, want %v", path, got, want)
+ }
+ }
+}
+
+func without(o fixtures.Object, key string) fixtures.Object {
+ var out fixtures.Object
+ for _, m := range o {
+ if m.Key != key {
+ out = append(out, m)
+ }
+ }
+ return out
+}
+
+// answer compares one answer; the chosen label is held to Python's only when it was decisive.
+func (c *tally) answer(path string, got, want fixtures.Object) {
+ if typ, _ := want.Get("type"); typ == "choice" {
+ var ps []float64
+ for _, m := range fixtures.Obj(c.t, want.Must("probabilities")) {
+ ps = append(ps, fixtures.Float(c.t, m.Value))
+ }
+ sort.Sort(sort.Reverse(sort.Float64Slice(ps)))
+ if len(ps) > 1 && ps[0]-ps[1] <= decisiveGap {
+ c.ties++
+ c.value(path, "", without(got, "choice"), without(want, "choice"))
+ return
+ }
+ c.labels++
+ if got.Must("choice") != want.Must("choice") {
+ c.labelDiffs++
+ }
+ }
+ c.value(path, "", got, want)
+}
+
+// prediction compares one result: model and usage exactly, answers by the rules above.
+func (c *tally) prediction(path string, p schema.Prediction, want fixtures.Object) {
+ data, err := json.Marshal(p)
+ if err != nil {
+ c.t.Fatal(err)
+ }
+ parsed, err := fixtures.Parse(data)
+ if err != nil {
+ c.t.Fatal(err)
+ }
+ got := fixtures.Obj(c.t, parsed)
+ c.value(path+".model", "model", got.Must("model"), want.Must("model"))
+ if g, w := fixtures.Marshal(c.t, got.Must("usage")), fixtures.Marshal(c.t, want.Must("usage")); string(g) != string(w) {
+ c.failf("%s usage\n got %s\nwant %s", path, g, w)
+ }
+ gotAnswers, wantAnswers := fixtures.Obj(c.t, got.Must("answers")), fixtures.Obj(c.t, want.Must("answers"))
+ if !reflect.DeepEqual(gotAnswers.Keys(), wantAnswers.Keys()) {
+ c.failf("%s answer ids %v, want %v", path, gotAnswers.Keys(), wantAnswers.Keys())
+ return
+ }
+ for _, m := range wantAnswers {
+ g, _ := gotAnswers.Get(m.Key)
+ c.answer(path+"."+m.Key, fixtures.Obj(c.t, g), fixtures.Obj(c.t, m.Value))
+ }
+}
+
+func (c *tally) report(name string) {
+ c.t.Logf("%s: %d numbers compared, %d differ at all, worst deviation %.2e, %d labels checked (%d differ), %d near-ties skipped",
+ name, c.numbers, c.differing, c.worst, c.labels, c.labelDiffs, c.ties)
+}
+
+func (c *tally) finish(name string, minNumbers int) {
+ c.report(name)
+ if c.numbers < minNumbers {
+ c.t.Errorf("%s: only %d numbers were compared", name, c.numbers)
+ }
+ if float64(c.differing) > maxDiffering*float64(c.numbers) {
+ c.t.Errorf("%s: %d of %d numbers differ (more than %.0f%%)", name, c.differing, c.numbers, maxDiffering*100)
+ }
+}
+
+func openAgent(t testing.TB, p paths, s *Session) *agent.Agent {
+ t.Helper()
+ a, err := agent.Open(os.DirFS(filepath.Join(p.checkpoints, "multilingual")), keepOpen{s})
+ if err != nil {
+ t.Fatalf("agent.Open: %v", err)
+ }
+ t.Cleanup(func() { a.Close() })
+ return a
+}
+
+type singleCase struct {
+ name string
+ state any
+ lang string
+ qs schema.Questions
+ want fixtures.Object
+}
+
+func loadState(t testing.TB, v any) any {
+ t.Helper()
+ st, err := sequence.ParseState(fixtures.Marshal(t, v))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return st
+}
+
+func loadQuestions(t testing.TB, v any) schema.Questions {
+ t.Helper()
+ qs, err := schema.ParseQuestions(fixtures.Marshal(t, v))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return qs
+}
+
+func singles(t testing.TB) []singleCase {
+ t.Helper()
+ golden := fixtures.Load(t, "predict.json")
+ var out []singleCase
+ for _, m := range fixtures.Obj(t, golden.Must("single")) {
+ rec := fixtures.Obj(t, m.Value)
+ out = append(out, singleCase{
+ name: m.Key, state: loadState(t, rec.Must("state")), lang: fixtures.StrOrEmpty(t, rec.Must("lang")),
+ qs: loadQuestions(t, rec.Must("questions")), want: rec,
+ })
+ }
+ return out
+}
+
+func (sc singleCase) options() []agent.CallOption {
+ if sc.lang == "" {
+ return nil
+ }
+ return []agent.CallOption{agent.WithLanguage(sc.lang)}
+}
+
+// TestPredictGolden holds the whole path (agent, collator, this session, ONNX Runtime) to the
+// answers Python's ONNXAgent recorded on the same graph.
+func TestPredictGolden(t *testing.T) {
+ p := need(t)
+ a := openAgent(t, p, sharedSession(t))
+ golden := fixtures.Load(t, "predict.json")
+ if cp := fixtures.Str(t, golden.Must("checkpoint")); cp != "multilingual" {
+ t.Fatalf("the golden was recorded on the %s checkpoint", cp)
+ }
+ ctx := context.Background()
+
+ total := &tally{t: t}
+ t.Run("single", func(t *testing.T) {
+ cases := singles(t)
+ if len(cases) != 6 {
+ t.Fatalf("%d single cases", len(cases))
+ }
+ c := &tally{t: t}
+ for _, sc := range cases {
+ c.where = sc.name
+ pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)
+ if err != nil {
+ t.Fatalf("%s: %v", sc.name, err)
+ }
+ c.prediction(sc.name, pred, sc.want)
+ }
+ c.finish("single", 60)
+ merge(total, c)
+ })
+
+ t.Run("batch", func(t *testing.T) {
+ batches := fixtures.List(t, golden.Must("batch"))
+ if len(batches) != 3 {
+ t.Fatalf("%d batch configurations", len(batches))
+ }
+ for _, raw := range batches {
+ rec := fixtures.Obj(t, raw)
+ var opts []agent.CallOption
+ name := "all-in-one"
+ if n := rec.Must("batch_size"); n != nil {
+ size := fixtures.Int(t, n)
+ opts = append(opts, agent.WithBatchSize(size))
+ name = "batch-" + string(rune('0'+size))
+ }
+ if fixtures.Bool(t, rec.Must("sort_by_length")) {
+ opts = append(opts, agent.WithSortByLength(true))
+ name += "-sorted"
+ }
+ t.Run(name, func(t *testing.T) {
+ var states []any
+ for _, s := range fixtures.List(t, rec.Must("states")) {
+ states = append(states, loadState(t, s))
+ }
+ preds, err := a.PredictBatch(ctx, states, loadQuestions(t, rec.Must("questions")), opts...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := fixtures.List(t, rec.Must("results"))
+ if len(preds) != len(want) || len(want) != 9 {
+ t.Fatalf("%d results, golden has %d", len(preds), len(want))
+ }
+ c := &tally{t: t}
+ for i, pred := range preds {
+ c.where = name
+ c.prediction(name+"/state"+string(rune('0'+i)), pred, fixtures.Obj(t, want[i]))
+ }
+ c.finish(name, 80)
+ merge(total, c)
+ })
+ }
+ })
+ total.report("all")
+ if total.labelDiffs != 0 {
+ t.Errorf("%d decisive labels differ from Python", total.labelDiffs)
+ }
+}
+
+func merge(dst, src *tally) {
+ dst.numbers += src.numbers
+ dst.differing += src.differing
+ dst.worst = math.Max(dst.worst, src.worst)
+ dst.labels += src.labels
+ dst.labelDiffs += src.labelDiffs
+ dst.ties += src.ties
+}
+
+// TestConcurrentPredict calls one agent from eight goroutines at once; each answer must equal
+// the serial one, and -race watches the Go side.
+func TestConcurrentPredict(t *testing.T) {
+ p := need(t)
+ a := openAgent(t, p, sharedSession(t))
+ cases := singles(t)
+ ctx := context.Background()
+ serial := make([][]byte, len(cases))
+ for i, sc := range cases {
+ pred, err := a.Predict(ctx, sc.state, sc.qs, sc.options()...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ serial[i], _ = json.Marshal(pred)
+ }
+ var wg sync.WaitGroup
+ errs := make(chan error, 8*len(cases))
+ for g := 0; g < 8; g++ {
+ wg.Add(1)
+ go func(g int) {
+ defer wg.Done()
+ for k := range cases {
+ i := (k + g) % len(cases)
+ pred, err := a.Predict(ctx, cases[i].state, cases[i].qs, cases[i].options()...)
+ if err != nil {
+ errs <- err
+ continue
+ }
+ if got, _ := json.Marshal(pred); string(got) != string(serial[i]) {
+ errs <- errors.New(cases[i].name + ": concurrent answer differs from the serial one:\n" + string(got) + "\n" + string(serial[i]))
+ }
+ }
+ }(g)
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ t.Error(err)
+ }
+}
+
+func TestPredictWithCancelledContext(t *testing.T) {
+ p := need(t)
+ s := sharedSession(t)
+ a := openAgent(t, p, s)
+ sc := singles(t)[0]
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ before := s.runs.Load()
+ if _, err := a.Predict(ctx, sc.state, sc.qs); !errors.Is(err, context.Canceled) {
+ t.Fatalf("Predict = %v, want context.Canceled", err)
+ }
+ if s.runs.Load() != before {
+ t.Fatal("the model ran for a cancelled context")
+ }
+}
+
+func TestRuntimeVersion(t *testing.T) {
+ s := sharedSession(t)
+ t.Logf("ONNX Runtime %s", s.RuntimeVersion())
+ if s.RuntimeVersion() == "" {
+ t.Fatal("empty RuntimeVersion")
+ }
+}
+
+// TestMarkerMaskReachesTheGraph pins that marker_mask is sent as given: padded slots must come
+// back different from real ones, so a session that sent all true would be caught.
+func TestMarkerMaskReachesTheGraph(t *testing.T) {
+ s := sharedSession(t)
+ masked := makeBatch(2, 16, 4)
+ for r := 0; r < masked.Rows; r++ {
+ masked.MarkerMask[r*4+2], masked.MarkerMask[r*4+3] = false, false
+ }
+ open := makeBatch(2, 16, 4)
+ a, err := s.Run(context.Background(), masked)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := s.Run(context.Background(), open)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("masked %v\nopen %v", a.Logits, b.Logits)
+ same := true
+ for i := range a.Logits {
+ same = same && a.Logits[i] == b.Logits[i]
+ }
+ if same {
+ t.Fatal("marker_mask has no effect on the logits")
+ }
+ if !masked.MarkerMask[0] || masked.MarkerMask[2] {
+ t.Fatal("Run modified the batch")
+ }
+}
+
+// TestTemperaturesReachTheAnswers pins that the checkpoint's temperatures are applied by type and
+// by option bucket. The shipped multilingual checkpoint uses 1.0 everywhere, which hides a
+// decode that ignores them, so this runs the same session under edited copies of its config.
+func TestTemperaturesReachTheAnswers(t *testing.T) {
+ p := need(t)
+ s := sharedSession(t)
+ src := filepath.Join(p.checkpoints, "multilingual")
+ raw, err := os.ReadFile(filepath.Join(src, "rl_agent_config.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sc := singles(t)[0]
+ if sc.name != "billing-mixed" {
+ t.Fatalf("first case is %s", sc.name)
+ }
+ answers := func(edit func(cfg map[string]any)) map[string]string {
+ var cfg map[string]any
+ if err := json.Unmarshal(raw, &cfg); err != nil {
+ t.Fatal(err)
+ }
+ edit(cfg)
+ dir := t.TempDir()
+ entries, err := os.ReadDir(src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, e := range entries {
+ if e.Name() != "rl_agent_config.json" {
+ if err := os.Symlink(filepath.Join(src, e.Name()), filepath.Join(dir, e.Name())); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ data, _ := json.Marshal(cfg)
+ if err := os.WriteFile(filepath.Join(dir, "rl_agent_config.json"), data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ a, err := agent.Open(os.DirFS(dir), keepOpen{s})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer a.Close()
+ pred, err := a.Predict(context.Background(), sc.state, sc.qs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, _ = json.Marshal(pred)
+ parsed, err := fixtures.Parse(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := map[string]string{}
+ for _, m := range fixtures.Obj(t, fixtures.Field(t, parsed, "answers")) {
+ out[m.Key] = string(fixtures.Marshal(t, m.Value))
+ }
+ return out
+ }
+ base := answers(func(map[string]any) {})
+ if len(base) != 3 {
+ t.Fatalf("base answers: %v", base)
+ }
+ ids := map[string]string{"choice": "intent", "score": "urgency", "noul": "duplicate"}
+ for i, typ := range []string{"choice", "score", "noul"} {
+ temps := []any{1.0, 1.0, 1.0}
+ temps[i] = 4.0
+ got := answers(func(c map[string]any) { c["temperature"] = temps })
+ for _, other := range []string{"choice", "score", "noul"} {
+ changed := got[ids[other]] != base[ids[other]]
+ if changed != (other == typ) {
+ t.Errorf("temperature %v: %s answer changed = %v", temps, other, changed)
+ }
+ }
+ }
+ got := answers(func(c map[string]any) { c["temperature_by_options"] = map[string]any{"choice:3-5": 4.0} })
+ if got["intent"] == base["intent"] || got["urgency"] != base["urgency"] || got["duplicate"] != base["duplicate"] {
+ t.Errorf("a choice:3-5 bucket temperature did not change exactly the 3-option choice: %v", got)
+ }
+}
diff --git a/laya-go/onnx/runtime.go b/laya-go/onnx/runtime.go
new file mode 100644
index 00000000..abb163c0
--- /dev/null
+++ b/laya-go/onnx/runtime.go
@@ -0,0 +1,69 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "fmt"
+ "os"
+ "sync"
+
+ ort "github.com/yalue/onnxruntime_go"
+)
+
+// The binding's environment is process-global and its init and destroy are unsynchronized, so
+// this file is the only place that calls them, behind one mutex.
+var rt struct {
+ mu sync.Mutex
+ refs int // sessions holding the environment
+ owned bool // we initialized it, so the last release destroys it
+ path string // the library we loaded, when owned
+ version string
+}
+
+// acquire takes a reference on the runtime, starting it on first use, and returns its version.
+func acquire(explicit string) (string, error) {
+ rt.mu.Lock()
+ defer rt.mu.Unlock()
+ if rt.refs > 0 {
+ if rt.owned && explicit != "" && !samePath(explicit, rt.path) {
+ return "", fmt.Errorf("%w: %q is loaded, %q was requested", ErrRuntimeConflict, rt.path, explicit)
+ }
+ rt.refs++
+ return rt.version, nil
+ }
+ if ort.IsInitialized() {
+ rt.owned, rt.path, rt.version = false, "", ort.GetVersion()
+ rt.refs = 1
+ return rt.version, nil
+ }
+ path := libraryPath(explicit, os.Getenv(LibraryEnv))
+ ort.SetSharedLibraryPath(path)
+ if err := ort.InitializeEnvironment(); err != nil {
+ return "", loadError(path, err)
+ }
+ rt.owned, rt.path, rt.version, rt.refs = true, path, ort.GetVersion(), 1
+ return rt.version, nil
+}
+
+// release drops a reference and unloads the runtime when it was the last and we loaded it.
+func release() error {
+ rt.mu.Lock()
+ defer rt.mu.Unlock()
+ rt.refs--
+ if rt.refs > 0 {
+ return nil
+ }
+ owned := rt.owned
+ rt.owned, rt.path, rt.version = false, "", ""
+ if owned {
+ return ort.DestroyEnvironment()
+ }
+ return nil
+}
+
+// references reports the sessions holding the runtime, for tests.
+func references() int {
+ rt.mu.Lock()
+ defer rt.mu.Unlock()
+ return rt.refs
+}
diff --git a/laya-go/onnx/session_cgo.go b/laya-go/onnx/session_cgo.go
new file mode 100644
index 00000000..8d96aa2c
--- /dev/null
+++ b/laya-go/onnx/session_cgo.go
@@ -0,0 +1,235 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "sync/atomic"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+ ort "github.com/yalue/onnxruntime_go"
+)
+
+// Session runs a fused laya graph on ONNX Runtime's CPU provider. It is safe for concurrent use.
+type Session struct {
+ mu sync.RWMutex // read-held by Run, write-held by Close
+ closed bool
+ s *ort.DynamicAdvancedSession
+ version string
+ runs atomic.Int64 // model runs started, for tests
+}
+
+var _ infer.Session = (*Session)(nil)
+
+var (
+ inputNames = []string{infer.NameInputIDs, infer.NameAttentionMask, infer.NameMarkerPos, infer.NameMarkerMask, infer.NameQType}
+ outputNames = []string{infer.NameLogits, infer.NameActLogits}
+)
+
+// Open loads a fused laya.onnx, whose .data sidecar must sit beside it, on the CPU provider.
+// A graph that is missing, split, or declares other tensors is a *GraphError.
+func Open(graphPath string, opts ...Option) (*Session, error) {
+ o, err := resolve(opts)
+ if err != nil {
+ return nil, err
+ }
+ if err := checkPath(graphPath); err != nil {
+ return nil, err
+ }
+ version, err := acquire(o.library)
+ if err != nil {
+ return nil, err
+ }
+ s, err := load(graphPath, o)
+ if err != nil {
+ return nil, errors.Join(err, release())
+ }
+ s.version = version
+ return s, nil
+}
+
+func load(path string, o options) (*Session, error) {
+ in, out, err := ort.GetInputOutputInfo(path)
+ if err != nil {
+ return nil, fmt.Errorf("onnx: cannot read %s: %w", path, err)
+ }
+ if err := checkGraph(infos(in), infos(out)); err != nil {
+ var ge *GraphError
+ if errors.As(err, &ge) {
+ ge.Path = path
+ }
+ return nil, err
+ }
+ so, err := ort.NewSessionOptions()
+ if err != nil {
+ return nil, fmt.Errorf("onnx: session options: %w", err)
+ }
+ defer so.Destroy()
+ if err := so.SetGraphOptimizationLevel(ort.GraphOptimizationLevelEnableAll); err != nil {
+ return nil, fmt.Errorf("onnx: optimization level: %w", err)
+ }
+ if o.intra > 0 {
+ if err := so.SetIntraOpNumThreads(o.intra); err != nil {
+ return nil, fmt.Errorf("onnx: intra-op threads: %w", err)
+ }
+ }
+ if o.inter > 0 {
+ if err := so.SetInterOpNumThreads(o.inter); err != nil {
+ return nil, fmt.Errorf("onnx: inter-op threads: %w", err)
+ }
+ }
+ ds, err := ort.NewDynamicAdvancedSession(path, inputNames, outputNames, so)
+ if err != nil {
+ return nil, fmt.Errorf("onnx: cannot load %s (its .data sidecar must sit beside it): %w", path, err)
+ }
+ return &Session{s: ds}, nil
+}
+
+func infos(in []ort.InputOutputInfo) []tensorInfo {
+ out := make([]tensorInfo, len(in))
+ for i, t := range in {
+ out[i] = tensorInfo{Name: t.Name, Dims: []int64(t.Dimensions), DType: "non-tensor"}
+ if t.OrtValueType != ort.ONNXTypeTensor {
+ continue
+ }
+ switch t.DataType {
+ case ort.TensorElementDataTypeInt64:
+ out[i].DType = "int64"
+ case ort.TensorElementDataTypeBool:
+ out[i].DType = "bool"
+ case ort.TensorElementDataTypeFloat:
+ out[i].DType = "float32"
+ default:
+ out[i].DType = t.DataType.String()
+ }
+ }
+ return out
+}
+
+// RuntimeVersion is the version of the ONNX Runtime library in use, such as "1.30.0".
+func (s *Session) RuntimeVersion() string { return s.version }
+
+// Close waits for runs in flight, releases the graph and drops its hold on ONNX Runtime, which
+// is unloaded when the last session closes. It is idempotent.
+func (s *Session) Close() error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.closed {
+ return nil
+ }
+ s.closed = true
+ err := s.s.Destroy()
+ s.s = nil
+ return errors.Join(err, release())
+}
+
+// Run runs one batch. A context that ends mid-run terminates the run and Run returns an error
+// wrapping ctx.Err(). Inputs are not modified or retained, and the returned slices are the
+// caller's.
+func (s *Session) Run(ctx context.Context, b infer.Batch) (infer.Output, error) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ if s.closed {
+ return infer.Output{}, infer.ErrClosed
+ }
+ if err := b.Validate(); err != nil {
+ return infer.Output{}, err
+ }
+ if err := ctx.Err(); err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: run not started: %w", err)
+ }
+ if b.Rows == 0 {
+ return infer.Output{}, nil
+ }
+
+ var vals []ort.Value
+ defer func() {
+ for _, v := range vals {
+ if v != nil {
+ v.Destroy()
+ }
+ }
+ }()
+ rows, length, markers := int64(b.Rows), int64(b.Length), int64(b.Markers)
+ ids, err := ort.NewTensor(ort.NewShape(rows, length), b.InputIDs)
+ if err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameInputIDs, err)
+ }
+ vals = append(vals, ids)
+ mask, err := ort.NewTensor(ort.NewShape(rows, length), b.AttentionMask)
+ if err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameAttentionMask, err)
+ }
+ vals = append(vals, mask)
+ pos, err := ort.NewTensor(ort.NewShape(rows, markers), b.MarkerPos)
+ if err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameMarkerPos, err)
+ }
+ vals = append(vals, pos)
+ mm, err := ort.NewTensor(ort.NewShape(rows, markers), b.MarkerMask)
+ if err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameMarkerMask, err)
+ }
+ vals = append(vals, mm)
+ qt, err := ort.NewTensor(ort.NewShape(rows), b.QType)
+ if err != nil {
+ return infer.Output{}, fmt.Errorf("onnx: %s tensor: %w", infer.NameQType, err)
+ }
+ vals = append(vals, qt)
+ inputs := append([]ort.Value(nil), vals...)
+
+ outputs := []ort.Value{nil, nil}
+ defer func() {
+ for _, v := range outputs {
+ if v != nil {
+ v.Destroy()
+ }
+ }
+ }()
+
+ s.runs.Add(1)
+ if err := s.execute(ctx, inputs, outputs); err != nil {
+ if cerr := ctx.Err(); cerr != nil {
+ return infer.Output{}, fmt.Errorf("onnx: run cancelled: %w", cerr)
+ }
+ return infer.Output{}, fmt.Errorf("onnx: run failed on %d rows of %d tokens: %w", b.Rows, b.Length, err)
+ }
+ logits, lok := outputs[0].(*ort.Tensor[float32])
+ act, aok := outputs[1].(*ort.Tensor[float32])
+ if !lok || !aok {
+ return infer.Output{}, fmt.Errorf("onnx: outputs are %T and %T, want float32 tensors", outputs[0], outputs[1])
+ }
+ return collect(b, outTensor{logits.GetShape(), logits.GetData()}, outTensor{act.GetShape(), act.GetData()})
+}
+
+// execute runs the graph, terminating it if ctx ends first. The watcher is always joined before
+// the run options are destroyed.
+func (s *Session) execute(ctx context.Context, inputs, outputs []ort.Value) error {
+ done := ctx.Done()
+ if done == nil {
+ return s.s.Run(inputs, outputs)
+ }
+ ro, err := ort.NewRunOptions()
+ if err != nil {
+ return fmt.Errorf("run options: %w", err)
+ }
+ defer ro.Destroy()
+ stop := make(chan struct{})
+ joined := make(chan struct{})
+ go func() {
+ defer close(joined)
+ select {
+ case <-done:
+ ro.Terminate()
+ case <-stop:
+ }
+ }()
+ defer func() {
+ close(stop)
+ <-joined
+ }()
+ return s.s.RunWithOptions(inputs, outputs, ro)
+}
diff --git a/laya-go/onnx/session_nocgo.go b/laya-go/onnx/session_nocgo.go
new file mode 100644
index 00000000..41b57a9a
--- /dev/null
+++ b/laya-go/onnx/session_nocgo.go
@@ -0,0 +1,28 @@
+//go:build !cgo
+
+package onnx
+
+import (
+ "context"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+// Session is the ONNX Runtime session. Without cgo it cannot be opened.
+type Session struct{}
+
+var _ infer.Session = (*Session)(nil)
+
+// Open returns ErrNoCGO: this build has no ONNX Runtime binding.
+func Open(graphPath string, opts ...Option) (*Session, error) { return nil, ErrNoCGO }
+
+// Run returns ErrNoCGO.
+func (s *Session) Run(ctx context.Context, b infer.Batch) (infer.Output, error) {
+ return infer.Output{}, ErrNoCGO
+}
+
+// Close does nothing.
+func (s *Session) Close() error { return nil }
+
+// RuntimeVersion returns "".
+func (s *Session) RuntimeVersion() string { return "" }
diff --git a/laya-go/onnx/support_test.go b/laya-go/onnx/support_test.go
new file mode 100644
index 00000000..238f162e
--- /dev/null
+++ b/laya-go/onnx/support_test.go
@@ -0,0 +1,104 @@
+//go:build cgo
+
+package onnx
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/infer"
+)
+
+// paths holds the three locations the model-backed tests need.
+type paths struct{ graph, library, checkpoints string }
+
+// need returns the model-backed tests' inputs, skipping when one is unset. CI turns a skip into
+// a failure.
+func need(t testing.TB) paths {
+ t.Helper()
+ p := paths{os.Getenv("LAYA_ONNX_GRAPH"), os.Getenv(LibraryEnv), os.Getenv("LAYA_CHECKPOINTS")}
+ if p.graph == "" || p.library == "" || p.checkpoints == "" {
+ t.Skip("set LAYA_ONNX_GRAPH, LAYA_ONNXRUNTIME_LIBRARY and LAYA_CHECKPOINTS")
+ }
+ return p
+}
+
+// sibling is another export beside the main one: <exports>/<name>/laya.onnx.
+func (p paths) sibling(name string) string {
+ return filepath.Join(filepath.Dir(filepath.Dir(p.graph)), name, "laya.onnx")
+}
+
+// The graph is 1.29 GB, so tests that are not about lifecycle share one session. Lifecycle tests
+// call closeShared first, so they start with no runtime loaded.
+var shared struct {
+ mu sync.Mutex
+ s *Session
+}
+
+func sharedSession(t testing.TB) *Session {
+ t.Helper()
+ p := need(t)
+ shared.mu.Lock()
+ defer shared.mu.Unlock()
+ if shared.s == nil {
+ s, err := Open(p.graph)
+ if err != nil {
+ t.Fatalf("Open(%s): %v", p.graph, err)
+ }
+ shared.s = s
+ }
+ return shared.s
+}
+
+func closeShared(t testing.TB) {
+ t.Helper()
+ shared.mu.Lock()
+ defer shared.mu.Unlock()
+ if shared.s != nil {
+ if err := shared.s.Close(); err != nil {
+ t.Fatalf("closing the shared session: %v", err)
+ }
+ shared.s = nil
+ }
+ if n := references(); n != 0 {
+ t.Fatalf("%d references to the runtime remain", n)
+ }
+}
+
+func TestMain(m *testing.M) {
+ code := m.Run()
+ shared.mu.Lock()
+ if shared.s != nil {
+ shared.s.Close()
+ }
+ shared.mu.Unlock()
+ os.Exit(code)
+}
+
+// keepOpen lets a test hand the shared session to agent.Open without the agent closing it.
+type keepOpen struct{ infer.Session }
+
+func (keepOpen) Close() error { return nil }
+
+// makeBatch builds a valid batch of rows x length tokens with the given marker slots.
+func makeBatch(rows, length, markers int) infer.Batch {
+ b := infer.Batch{
+ InputIDs: make([]int64, rows*length), AttentionMask: make([]int64, rows*length),
+ MarkerPos: make([]int64, rows*markers), MarkerMask: make([]bool, rows*markers),
+ QType: make([]int64, rows), Rows: rows, Length: length, Markers: markers,
+ }
+ for i := range b.InputIDs {
+ b.InputIDs[i] = int64(5 + i%97)
+ b.AttentionMask[i] = 1
+ }
+ for i := range b.MarkerPos {
+ b.MarkerPos[i] = int64(1 + i%markers*2%(length-1))
+ b.MarkerMask[i] = true
+ }
+ for i := range b.QType {
+ b.QType[i] = int64(i % 3)
+ }
+ return b
+}