Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

laya-go: unit tests for the confidence gate

Adeshina Lasisi7fc3d880069c
diff --git a/laya-go/agent/gate_internal_test.go b/laya-go/agent/gate_internal_test.go
new file mode 100644
index 00000000..a783aeaa
--- /dev/null
+++ b/laya-go/agent/gate_internal_test.go
@@ -0,0 +1,91 @@
+package agent
+
+import (
+ "math"
+ "strconv"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/internal/pyjson"
+ "github.com/NandhaKishorM/laya/laya-go/schema"
+)
+
+func probsOf(n int) schema.Probabilities {
+ ps := make(schema.Probabilities, n)
+ for i := range ps {
+ ps[i] = schema.Probability{Label: schema.Text(strconv.Itoa(i)), P: 1 / float64(n)}
+ }
+ return ps
+}
+
+func TestBucketEdges(t *testing.T) {
+ for k, want := range map[int]string{1: "choice:2", 2: "choice:2", 3: "choice:3-5", 5: "choice:3-5", 6: "choice:6-10", 10: "choice:6-10", 11: "choice:11+", 40: "choice:11+"} {
+ if got := bucketOf(&schema.ChoiceAnswer{Probabilities: probsOf(k)}); got != want {
+ t.Errorf("choice k=%d: %q, want %q", k, got, want)
+ }
+ }
+ if got := bucketOf(&schema.ScoreAnswer{Probabilities: probsOf(7)}); got != "score:6-10" {
+ t.Errorf("score: %q", got)
+ }
+ if got := bucketOf(&schema.NoulAnswer{}); got != "noul:2" {
+ t.Errorf("noul: %q", got)
+ }
+ if got := bucketOf(&schema.ChoiceAnswer{}); got != "" {
+ t.Errorf("a choice without probabilities has a bucket %q", got)
+ }
+}
+
+func TestValidThresholds(t *testing.T) {
+ for _, v := range []any{0.0, 1.0, 0.5, 0, 1} {
+ if _, err := checkMinConfidence(v); err != nil {
+ t.Errorf("%v refused: %v", v, err)
+ }
+ }
+ for _, v := range []any{math.Inf(1), math.Inf(-1), 2, -1, nil} {
+ if _, err := checkMinConfidence(v); err == nil {
+ t.Errorf("%v accepted", v)
+ }
+ }
+ ok := pyjson.Object{{Key: "default", Value: 0.3}, {Key: "noul:11+", Value: 0.2}}
+ if _, err := checkMinConfidence(ok); err != nil {
+ t.Errorf("valid map refused: %v", err)
+ }
+ for _, k := range []string{"choice:1", "choice", "other:2", "choice:3-5 ", "Default"} {
+ if _, err := checkMinConfidence(pyjson.Object{{Key: k, Value: 0.5}}); err == nil {
+ t.Errorf("bucket key %q accepted", k)
+ }
+ }
+}
+
+func TestMapFallsBackToDefaultThenZero(t *testing.T) {
+ a := &schema.ChoiceAnswer{Probabilities: probsOf(4)}
+ g, _ := checkMinConfidence(pyjson.Object{{Key: "default", Value: 0.4}})
+ if got := g.threshold(a); got != 0.4 {
+ t.Errorf("default: %v", got)
+ }
+ g, _ = checkMinConfidence(pyjson.Object{{Key: "choice:2", Value: 0.9}})
+ if got := g.threshold(a); got != 0 {
+ t.Errorf("no default: %v", got)
+ }
+}
+
+func TestStaleFlagIsCleared(t *testing.T) {
+ c := &schema.ChoiceAnswer{Probabilities: probsOf(2), Common: schema.Common{AnswerConfidence: 0.9, Confidence: 0.5, LowConfidence: true}}
+ preds := []schema.Prediction{{Answers: schema.Answers{{ID: "q", Answer: c}}}}
+ g, _ := checkMinConfidence(0.5)
+ g.apply(preds)
+ if c.LowConfidence || c.Abstention != gatePassed {
+ t.Fatalf("flag %v state %q", c.LowConfidence, c.Abstention)
+ }
+}
+
+func TestGateConfidencePrefersAnswerConfidence(t *testing.T) {
+ if v, ok := gateConfidence(schema.Common{AnswerConfidence: 0.2, Confidence: 0.9}); !ok || v != 0.2 {
+ t.Errorf("got %v %v", v, ok)
+ }
+ if v, ok := gateConfidence(schema.Common{AnswerConfidence: math.NaN(), Confidence: 0.9}); !ok || v != 0.9 {
+ t.Errorf("fallback got %v %v", v, ok)
+ }
+ if _, ok := gateConfidence(schema.Common{AnswerConfidence: math.NaN(), Confidence: math.Inf(1)}); ok {
+ t.Error("non-finite confidences usable")
+ }
+}
diff --git a/laya-go/agent/gate_test.go b/laya-go/agent/gate_test.go
new file mode 100644
index 00000000..acc11110
--- /dev/null
+++ b/laya-go/agent/gate_test.go
@@ -0,0 +1,117 @@
+package agent_test
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/agent"
+ "github.com/NandhaKishorM/laya/laya-go/schema"
+)
+
+const gateKeys = `"abstention":`
+
+func TestUngatedWritesNoGateField(t *testing.T) {
+ a, _ := open(t, `{}`)
+ p, err := a.Predict(context.Background(), "hello", questions(t))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if s := jsonOf(t, p); strings.Contains(s, "abstention") || strings.Contains(s, "low_confidence") {
+ t.Fatalf("an ungated answer carries a gate field: %s", s)
+ }
+}
+
+func TestGateStatesOnPredict(t *testing.T) {
+ a, _ := open(t, `{}`)
+ ctx := context.Background()
+ // 0 flags nothing and still reports the gate ran; 1 flags every answer short of certainty.
+ p0, err := a.Predict(ctx, "hello", questions(t), agent.WithMinConfidence(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if s := jsonOf(t, p0); strings.Count(s, `"abstention":"passed","abstention_threshold":0.0`) != 3 || strings.Contains(s, "low_confidence") {
+ t.Fatalf("threshold 0: %s", s)
+ }
+ p1, err := a.Predict(ctx, "hello", questions(t), agent.WithMinConfidence(1))
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := jsonOf(t, p1)
+ if strings.Count(s, `"low_confidence":true,"abstention":"abstained","abstention_threshold":1.0`) != 3 {
+ t.Fatalf("threshold 1: %s", s)
+ }
+}
+
+func TestGateAppliesToEveryStateOfABatch(t *testing.T) {
+ a, _ := open(t, `{}`)
+ ps, err := a.PredictBatch(context.Background(), []any{"one", "two"}, questions(t), agent.WithMinConfidence(1))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i, p := range ps {
+ if n := strings.Count(jsonOf(t, p), gateKeys); n != 3 {
+ t.Errorf("state %d: %d gated answers, want 3", i, n)
+ }
+ }
+}
+
+func TestGateMapPicksTheBucket(t *testing.T) {
+ a, _ := open(t, `{}`)
+ // choice has 3 options (3-5), score 3 levels (3-5), noul is 2. Only the choice bucket is named.
+ p, err := a.Predict(context.Background(), "hello", questions(t),
+ agent.WithMinConfidenceMap(map[string]float64{"choice:3-5": 1, "noul:2": 0.25}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := map[string]float64{"intent": 1, "urgency": 0, "angry": 0.25}
+ for id, thr := range want {
+ ans, _ := p.Answers.Get(id)
+ var c schema.Common
+ switch x := ans.(type) {
+ case *schema.ChoiceAnswer:
+ c = x.Common
+ case *schema.ScoreAnswer:
+ c = x.Common
+ case *schema.NoulAnswer:
+ c = x.Common
+ }
+ if c.AbstentionThreshold == nil || *c.AbstentionThreshold != thr {
+ t.Errorf("%s threshold = %v, want %v", id, c.AbstentionThreshold, thr)
+ }
+ }
+}
+
+func TestInvalidThresholdFailsBeforeAnyWork(t *testing.T) {
+ a, s := open(t, `{}`)
+ cancelled, cancel := context.WithCancel(context.Background())
+ cancel()
+ for _, thr := range []float64{1.5, -0.1} {
+ // A done context is reported after the threshold, as laya validates first.
+ _, err := a.PredictBatch(cancelled, []any{"x"}, questions(t), agent.WithMinConfidence(thr))
+ var me *agent.MinConfidenceError
+ if !errors.As(err, &me) {
+ t.Fatalf("threshold %v: err = %v, want *MinConfidenceError", thr, err)
+ }
+ if _, err := a.Predict(context.Background(), "x", questions(t), agent.WithMinConfidence(thr)); !errors.As(err, &me) {
+ t.Fatalf("Predict %v: err = %v", thr, err)
+ }
+ }
+ if len(s.seen()) != 0 {
+ t.Fatal("the session ran for an invalid threshold")
+ }
+}
+
+func TestPredictLongWritesNoGateField(t *testing.T) {
+ a, _ := open(t, `{}`)
+ // PredictLong has no gate: laya calls it with None, which writes nothing, so even a given
+ // option leaves the answers as ungated.
+ lp, err := a.PredictLong(context.Background(), "hello", questions(t), agent.WithMinConfidence(1))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if s := jsonOf(t, lp.Prediction); strings.Contains(s, "abstention") || strings.Contains(s, "low_confidence") {
+ t.Fatalf("PredictLong carries a gate field: %s", s)
+ }
+}
diff --git a/laya-go/schema/gate_test.go b/laya-go/schema/gate_test.go
new file mode 100644
index 00000000..3c46cfe3
--- /dev/null
+++ b/laya-go/schema/gate_test.go
@@ -0,0 +1,51 @@
+package schema_test
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/NandhaKishorM/laya/laya-go/schema"
+)
+
+func TestGateFieldsRoundTrip(t *testing.T) {
+ thr := 0.25
+ a := &schema.NoulAnswer{Noul: 0.6, Common: schema.Common{Confidence: 0.6, AnswerConfidence: 0.6, LowConfidence: true, Abstention: "abstained", AbstentionThreshold: &thr}}
+ b, err := json.Marshal(a)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := string(b)
+ if !strings.HasSuffix(s, `"low_confidence":true,"abstention":"abstained","abstention_threshold":0.25}`) {
+ t.Fatalf("gate fields not last, in order: %s", s)
+ }
+ back, err := schema.UnmarshalAnswer(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := back.(*schema.NoulAnswer).Common
+ if !c.LowConfidence || c.Abstention != "abstained" || c.AbstentionThreshold == nil || *c.AbstentionThreshold != 0.25 {
+ t.Fatalf("round trip lost the gate: %+v", c)
+ }
+}
+
+func TestGateFieldsAbsentWhenUngated(t *testing.T) {
+ b, _ := json.Marshal(&schema.NoulAnswer{Noul: 0.6, Common: schema.Common{Confidence: 0.6, AnswerConfidence: 0.6}})
+ if strings.Contains(string(b), "abstention") || strings.Contains(string(b), "low_confidence") {
+ t.Fatalf("%s", b)
+ }
+}
+
+func TestGateFieldsRefuseBadInput(t *testing.T) {
+ base := `{"type":"noul","probabilities":{"0":0.4,"1":0.6},"action":{"act_probability":0.5},"noul":0.6,"answer_confidence":0.6,"confidence":0.6`
+ for _, extra := range []string{
+ `,"abstention":"passed"`, // state without threshold
+ `,"abstention_threshold":0.5`, // threshold without state
+ `,"abstention":"maybe","abstention_threshold":0.5`, // unknown state
+ `,"low_confidence":false`, // laya removes the key, never writes false
+ } {
+ if _, err := schema.UnmarshalAnswer([]byte(base + extra + "}")); err == nil {
+ t.Errorf("accepted %s", extra)
+ }
+ }
+}