Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
router lifecycle: review fixes
Adeshina Lasisi47cc940e1c2b
diff --git a/laya-go/router/hooks.go b/laya-go/router/hooks.goindex 201626c5..b71263cf 100644--- a/laya-go/router/hooks.go+++ b/laya-go/router/hooks.go@@ -3,7 +3,8 @@ package router// Hooks are laya/hooks.py for the three events a Router raises: on_route, on_load and on_evict. A// hook is any value with an OnRoute, OnLoad or OnEvict method; the same value may also be an// agent.Hook, as in Python, where one object implements several events. OnLoad and OnEvict run-// after the Router's lock is released (see lifecycle.go), so a hook may call the Router. Each call of Route+// after the Router's lock is released (see lifecycle.go), so a hook may call the Router. As in Python, an Unload that races a build of the same checkpoint+// can fire OnEvict before that build's OnLoad. Each call of Route// dispatches over: the process-wide agent.DefaultHooks (which take only an agent.Hook, so a// hook that is both embeds agent.BaseHook), the Router's installed hooks, then the// call's ForHooks, and a hook without OnRoute is skipped.diff --git a/laya-go/router/lifecycle.go b/laya-go/router/lifecycle.goindex 1587342a..2a4a3a32 100644--- a/laya-go/router/lifecycle.go+++ b/laya-go/router/lifecycle.go@@ -32,7 +32,10 @@ var ErrClosed = errors.New("router: closed")// AgentFactory builds the agent for a checkpoint. It owns where the weights come from, the// device and the agent options. It is called at most once at a time per Router and never with a// lock held, and ctx is the context of the call that triggered the build. The Router closes the-// agent it returns, so a factory must return a fresh agent on every call, never one it shares.+// agent it returns, so a factory must return a fresh agent on every call, never one it shares. A+// factory must not call Load, Lease, Preload, Unload, UnloadAll or Close on the Router that is+// calling it: it holds the build token, so a call that builds, or waits for a build, waits for+// itself.type AgentFactory func(ctx context.Context, c Checkpoint) (*agent.Agent, error)// WithAgentFactory lets the Router load checkpoints. Without it Route works and every load fails.@@ -57,9 +60,15 @@ type slot struct {agent *agent.Agentleases intretired bool // left the Router- owns bool // the Router built it, so the Router closes it+ own *own // non-nil when the Router built the agent; Attach clears it}+// own is the Router's claim to close an agent it built. It is shared by every slot of that agent,+// so attaching the agent hands it to the caller even while an earlier, retired slot is leased.+type own struct{ router bool }++func (s *slot) owns() bool { return s.own != nil && s.own.router }+type inflight struct {done chan struct{}err error // set before done closes@@ -75,6 +84,7 @@ type lifecycle struct {loading map[Checkpoint]*inflightmax intclosed bool+ owners map[*agent.Agent]*own // agents the Router built and has not closed}func (l *lifecycle) init(f AgentFactory, max int) {@@ -82,6 +92,7 @@ func (l *lifecycle) init(f AgentFactory, max int) {l.build = make(chan struct{}, 1)l.slots = map[Checkpoint]*slot{}l.loading = map[Checkpoint]*inflight{}+ l.owners = map[*agent.Agent]*own{}}// outcome is what one acquire did, so its hooks fire after the locks are released.@@ -107,7 +118,8 @@ func (l *lifecycle) touch(key Checkpoint) {// it and the Router built it. mu is held.func (l *lifecycle) retire(s *slot, toClose *[]*agent.Agent) {s.retired = true- if s.leases <= 0 && s.owns {+ if s.leases <= 0 && s.owns() {+ delete(l.owners, s.agent)*toClose = append(*toClose, s.agent)}}@@ -127,10 +139,21 @@ func (l *lifecycle) evictLocked(toClose *[]*agent.Agent) []Checkpoint {return evicted}+// safeClose closes a, turning a panic in its session's Close into an error so one bad agent+// cannot skip the others or the events that follow.+func safeClose(a *agent.Agent) (err error) {+ defer func() {+ if r := recover(); r != nil {+ err = fmt.Errorf("router: closing an agent panicked: %v", r)+ }+ }()+ return a.Close()+}+func closeAll(agents []*agent.Agent) error {var errs []errorfor _, a := range agents {- if err := a.Close(); err != nil {+ if err := safeClose(a); err != nil {errs = append(errs, err)}}@@ -142,12 +165,13 @@ func (l *lifecycle) release(s *slot) error {var toClose *agent.Agentl.mu.Lock()s.leases--- if s.leases <= 0 && s.retired && s.owns {+ if s.leases <= 0 && s.retired && s.owns() {toClose = s.agent+ delete(l.owners, s.agent)}l.mu.Unlock()if toClose != nil {- return toClose.Close()+ return safeClose(toClose)}return nil}@@ -214,7 +238,12 @@ func (l *lifecycle) create(ctx context.Context, key Checkpoint) (*agent.Agent, e// doBuild is the build this caller owns: mine is the handle the others wait on.func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight, out *outcome) (built *slot, err error) {var toClose []*agent.Agent- finish := func() { // mu is held+ finished := false+ finish := func() { // mu is held; idempotent+ if finished {+ return+ }+ finished = trueif l.loading[key] == mine {delete(l.loading, key)}@@ -228,7 +257,9 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,if err != nil {built = nill.mu.Lock()- mine.err = err+ if mine.err == nil && !finished {+ mine.err = err+ }finish()l.mu.Unlock()}@@ -251,6 +282,10 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,defer releaseBuild()l.mu.Lock()+ if l.closed { // Close came while this caller waited for the token: do not start a build+ l.mu.Unlock()+ return nil, ErrClosed+ }if s := l.slots[key]; s != nil { // built or attached while this caller waited for the build tokenl.touch(key)s.leases++@@ -273,7 +308,9 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,toClose = append(toClose, a)out.built, out.agent = true, s.agent} else {- built = &slot{agent: a, owns: true, leases: 1}+ o := &own{router: true}+ l.owners[a] = o+ built = &slot{agent: a, own: o, leases: 1}l.slots[key] = builtl.order = append(l.order, key)out.evicted = l.evictLocked(&toClose)@@ -384,9 +421,10 @@ func (r *Router) Load(ctx context.Context, name string) (*agent.Agent, error) {if err != nil {return nil, err}- // After release, so a hook that calls back in does not find this call holding a lease.- rerr := r.lc.release(s)- if herr := r.dispatchOutcome(key, out); herr != nil || rerr != nil {+ // The lease is held across the hooks, so a concurrent eviction cannot close ctx.Agent under+ // OnLoad; it is given back before Load returns.+ herr := r.dispatchOutcome(key, out)+ if rerr := r.lc.release(s); herr != nil || rerr != nil {return nil, errors.Join(herr, rerr)}return s.agent, nil@@ -414,6 +452,10 @@ func (r *Router) Attach(name string, a *agent.Agent) error {return ErrClosed}prev := l.slots[key]+ if o := l.owners[a]; o != nil { // built here, perhaps evicted but still leased: now the caller's+ o.router = false+ delete(l.owners, a)+ }l.slots[key] = &slot{agent: a}if prev != nil && prev.agent != a { // the same agent changes hands, it is not closedl.retire(prev, &toClose)diff --git a/laya-go/router/lifecycle_review_test.go b/laya-go/router/lifecycle_review_test.gonew file mode 100644index 00000000..7c23e104--- /dev/null+++ b/laya-go/router/lifecycle_review_test.go@@ -0,0 +1,212 @@+package router_test++import (+ "context"+ "errors"+ "math/rand"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/NandhaKishorM/laya/laya-go/agent"+ "github.com/NandhaKishorM/laya/laya-go/router"+)++type panicSession struct{ probeSession }++func (s *panicSession) Close() error { panic("session close exploded") }++func TestPanicInEvictionCloseDoesNotWedgeTheRouter(t *testing.T) {+ var evicted []string+ var mu sync.Mutex+ first := true+ f := func(ctx context.Context, c router.Checkpoint) (*agent.Agent, error) {+ if first {+ first = false+ return agent.Open(toyCheckpoint(), &panicSession{})+ }+ return newProbe(t, nil), nil+ }+ h := &router.HookFuncs{OnEvictFunc: func(c *router.EvictContext) { mu.Lock(); evicted = append(evicted, string(c.Model)); mu.Unlock() }}+ r, err := router.New(router.WithAgentFactory(f), router.WithMaxLoaded(1), router.WithInstalledHooks(h))+ if err != nil {+ t.Fatal(err)+ }+ ctx := context.Background()+ if _, err := r.Load(ctx, "english"); err != nil {+ t.Fatal(err)+ }+ _, err = r.Lease(ctx, "multilingual") // evicts english, whose Close panics+ if err == nil || !strings.Contains(err.Error(), "session close exploded") {+ t.Fatalf("err = %v", err)+ }+ within(t, 2*time.Second, "Loaded after a panicking close", func() { r.Loaded() })+ mu.Lock()+ if len(evicted) != 1 || evicted[0] != "english" {+ t.Fatalf("on_evict events %v: the panic skipped them", evicted)+ }+ mu.Unlock()+ within(t, 2*time.Second, "next Load", func() {+ if _, err := r.Load(ctx, "multilingual"); err != nil {+ t.Error(err)+ }+ })+}++func TestPanicInOneCloseDoesNotSkipTheOthers(t *testing.T) {+ good := &probeSession{}+ n := 0+ f := func(ctx context.Context, c router.Checkpoint) (*agent.Agent, error) {+ n+++ if n == 1 {+ return agent.Open(toyCheckpoint(), &panicSession{})+ }+ return newProbe(t, good), nil+ }+ r, _ := router.New(router.WithAgentFactory(f), router.WithMaxLoaded(3))+ ctx := context.Background()+ r.Load(ctx, "english")+ r.Load(ctx, "multilingual")+ if _, err := r.UnloadAll(ctx); err == nil {+ t.Fatal("the panic was lost")+ }+ if good.closes.Load() != 1 {+ t.Fatal("a panicking close skipped the next agent")+ }+}++func TestAttachOfAnEvictedLeasedAgentIsNeverClosedByTheRouter(t *testing.T) {+ f := newFactory(t)+ r := newRouter(t, f, router.WithMaxLoaded(1))+ ls, err := r.Lease(context.Background(), "english")+ if err != nil {+ t.Fatal(err)+ }+ a := ls.Agent()+ mustLoad(t, r, "multilingual") // evicts english while leased+ if err := r.Attach("english", a); err != nil {+ t.Fatal(err)+ }+ ls.Close()+ if n := f.of(a).sess.closes.Load(); n != 0 {+ t.Fatalf("an attached agent was closed by the Router (closes=%d)", n)+ }+}++func TestLoadKeepsItsLeaseAcrossTheLoadHook(t *testing.T) {+ f := newFactory(t)+ r := newRouter(t, f, router.WithMaxLoaded(1))+ var sawClosed atomic.Bool+ var once sync.Once+ r.AddHook(&router.HookFuncs{OnLoadFunc: func(c *router.LoadContext) {+ if c.Model != router.English {+ return+ }+ once.Do(func() {+ go r.Load(context.Background(), "multilingual") // evicts english under the hook+ b := f.of(c.Agent)+ for i := 0; i < 100 && b.sess.closes.Load() == 0; i++ {+ time.Sleep(5 * time.Millisecond)+ }+ sawClosed.Store(b.sess.closes.Load() > 0)+ })+ }})+ mustLoad(t, r, "english")+ if sawClosed.Load() {+ t.Fatal("OnLoad held an agent that a concurrent eviction closed")+ }+}++func TestFactoryNotCalledForABuildQueuedBehindClose(t *testing.T) {+ f := newFactory(t)+ f.entered = make(chan router.Checkpoint, 4)+ r := newRouter(t, f)+ gate := f.open(en)+ go r.Load(context.Background(), "english")+ <-f.entered+ qerr := make(chan error, 1)+ go func() { _, err := r.Load(context.Background(), "multilingual"); qerr <- err }() // queued on the token+ time.Sleep(30 * time.Millisecond)+ cdone := make(chan struct{})+ go func() { r.Close(); close(cdone) }()+ time.Sleep(30 * time.Millisecond)+ close(gate)+ <-cdone+ if err := <-qerr; !errors.Is(err, router.ErrClosed) {+ t.Fatalf("queued build: %v", err)+ }+ if f.calls() != 1 {+ t.Fatalf("factory called %d times; a build started after Close", f.calls())+ }+}++func TestBuildQueuedBehindTheTokenSeesAnAttach(t *testing.T) {+ f := newFactory(t)+ f.entered = make(chan router.Checkpoint, 4)+ gate := f.open(en)+ r := newRouter(t, f)+ go r.Load(context.Background(), "english")+ <-f.entered+ res := make(chan *agent.Agent, 1)+ go func() { a, _ := r.Load(context.Background(), "multilingual"); res <- a }()+ time.Sleep(30 * time.Millisecond)+ attached := newProbe(t, nil)+ if err := r.Attach("multilingual", attached); err != nil {+ t.Fatal(err)+ }+ close(gate)+ if got := <-res; got != attached {+ t.Fatal("the queued build did not use the attached agent")+ }+ if f.calls() != 1 {+ t.Fatalf("factory called %d times for a checkpoint that was attached", f.calls())+ }+}++func TestStressLeaseAccountingWithCancellation(t *testing.T) {+ f := newFactory(t)+ r := newRouter(t, f, router.WithMaxLoaded(1))+ names := []string{"english", "multilingual", "typed-decisions"}+ var wg sync.WaitGroup+ for g := 0; g < 8; g++ {+ wg.Add(1)+ go func(seed int64) {+ defer wg.Done()+ rnd := rand.New(rand.NewSource(seed))+ for i := 0; i < 400; i++ {+ n := names[rnd.Intn(3)]+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1+rnd.Intn(20))*time.Millisecond)+ switch rnd.Intn(5) {+ case 0, 1:+ if l, err := r.Lease(ctx, n); err == nil {+ if l.Agent() == nil || f.of(l.Agent()).sess.closes.Load() != 0 {+ t.Error("lease handed out a closed agent")+ }+ l.Close()+ l.Close()+ }+ case 2:+ r.Load(ctx, n)+ case 3:+ r.Unload(ctx, n)+ default:+ r.UnloadAll(ctx)+ }+ cancel()+ }+ }(int64(g))+ }+ wg.Wait()+ if err := r.Close(); err != nil {+ t.Fatal(err)+ }+ f.mu.Lock()+ defer f.mu.Unlock()+ for _, b := range f.made {+ if c := b.sess.closes.Load(); c != 1 {+ t.Errorf("%s closed %d times", b.label, c)+ }+ }+}