Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

router lifecycle: review fixes

Adeshina Lasisi47cc940e1c2b
diff --git a/laya-go/router/hooks.go b/laya-go/router/hooks.go
index 201626c5..b71263cf 100644
--- a/laya-go/router/hooks.go
+++ b/laya-go/router/hooks.go
@@ -3,7 +3,8 @@ package router
// Hooks are laya/hooks.py for the three events a Router raises: on_route, on_load and on_evict. A
// hook is any value with an OnRoute, OnLoad or OnEvict method; the same value may also be an
// agent.Hook, as in Python, where one object implements several events. OnLoad and OnEvict run
-// after the Router's lock is released (see lifecycle.go), so a hook may call the Router. Each call of Route
+// after the Router's lock is released (see lifecycle.go), so a hook may call the Router. As in Python, an Unload that races a build of the same checkpoint
+// can fire OnEvict before that build's OnLoad. Each call of Route
// dispatches over: the process-wide agent.DefaultHooks (which take only an agent.Hook, so a
// hook that is both embeds agent.BaseHook), the Router's installed hooks, then the
// call's ForHooks, and a hook without OnRoute is skipped.
diff --git a/laya-go/router/lifecycle.go b/laya-go/router/lifecycle.go
index 1587342a..2a4a3a32 100644
--- a/laya-go/router/lifecycle.go
+++ b/laya-go/router/lifecycle.go
@@ -32,7 +32,10 @@ var ErrClosed = errors.New("router: closed")
// AgentFactory builds the agent for a checkpoint. It owns where the weights come from, the
// device and the agent options. It is called at most once at a time per Router and never with a
// lock held, and ctx is the context of the call that triggered the build. The Router closes the
-// agent it returns, so a factory must return a fresh agent on every call, never one it shares.
+// agent it returns, so a factory must return a fresh agent on every call, never one it shares. A
+// factory must not call Load, Lease, Preload, Unload, UnloadAll or Close on the Router that is
+// calling it: it holds the build token, so a call that builds, or waits for a build, waits for
+// itself.
type AgentFactory func(ctx context.Context, c Checkpoint) (*agent.Agent, error)
// WithAgentFactory lets the Router load checkpoints. Without it Route works and every load fails.
@@ -57,9 +60,15 @@ type slot struct {
agent *agent.Agent
leases int
retired bool // left the Router
- owns bool // the Router built it, so the Router closes it
+ own *own // non-nil when the Router built the agent; Attach clears it
}
+// own is the Router's claim to close an agent it built. It is shared by every slot of that agent,
+// so attaching the agent hands it to the caller even while an earlier, retired slot is leased.
+type own struct{ router bool }
+
+func (s *slot) owns() bool { return s.own != nil && s.own.router }
+
type inflight struct {
done chan struct{}
err error // set before done closes
@@ -75,6 +84,7 @@ type lifecycle struct {
loading map[Checkpoint]*inflight
max int
closed bool
+ owners map[*agent.Agent]*own // agents the Router built and has not closed
}
func (l *lifecycle) init(f AgentFactory, max int) {
@@ -82,6 +92,7 @@ func (l *lifecycle) init(f AgentFactory, max int) {
l.build = make(chan struct{}, 1)
l.slots = map[Checkpoint]*slot{}
l.loading = map[Checkpoint]*inflight{}
+ l.owners = map[*agent.Agent]*own{}
}
// outcome is what one acquire did, so its hooks fire after the locks are released.
@@ -107,7 +118,8 @@ func (l *lifecycle) touch(key Checkpoint) {
// it and the Router built it. mu is held.
func (l *lifecycle) retire(s *slot, toClose *[]*agent.Agent) {
s.retired = true
- if s.leases <= 0 && s.owns {
+ if s.leases <= 0 && s.owns() {
+ delete(l.owners, s.agent)
*toClose = append(*toClose, s.agent)
}
}
@@ -127,10 +139,21 @@ func (l *lifecycle) evictLocked(toClose *[]*agent.Agent) []Checkpoint {
return evicted
}
+// safeClose closes a, turning a panic in its session's Close into an error so one bad agent
+// cannot skip the others or the events that follow.
+func safeClose(a *agent.Agent) (err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ err = fmt.Errorf("router: closing an agent panicked: %v", r)
+ }
+ }()
+ return a.Close()
+}
+
func closeAll(agents []*agent.Agent) error {
var errs []error
for _, a := range agents {
- if err := a.Close(); err != nil {
+ if err := safeClose(a); err != nil {
errs = append(errs, err)
}
}
@@ -142,12 +165,13 @@ func (l *lifecycle) release(s *slot) error {
var toClose *agent.Agent
l.mu.Lock()
s.leases--
- if s.leases <= 0 && s.retired && s.owns {
+ if s.leases <= 0 && s.retired && s.owns() {
toClose = s.agent
+ delete(l.owners, s.agent)
}
l.mu.Unlock()
if toClose != nil {
- return toClose.Close()
+ return safeClose(toClose)
}
return nil
}
@@ -214,7 +238,12 @@ func (l *lifecycle) create(ctx context.Context, key Checkpoint) (*agent.Agent, e
// doBuild is the build this caller owns: mine is the handle the others wait on.
func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight, out *outcome) (built *slot, err error) {
var toClose []*agent.Agent
- finish := func() { // mu is held
+ finished := false
+ finish := func() { // mu is held; idempotent
+ if finished {
+ return
+ }
+ finished = true
if l.loading[key] == mine {
delete(l.loading, key)
}
@@ -228,7 +257,9 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,
if err != nil {
built = nil
l.mu.Lock()
- mine.err = err
+ if mine.err == nil && !finished {
+ mine.err = err
+ }
finish()
l.mu.Unlock()
}
@@ -251,6 +282,10 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,
defer releaseBuild()
l.mu.Lock()
+ if l.closed { // Close came while this caller waited for the token: do not start a build
+ l.mu.Unlock()
+ return nil, ErrClosed
+ }
if s := l.slots[key]; s != nil { // built or attached while this caller waited for the build token
l.touch(key)
s.leases++
@@ -273,7 +308,9 @@ func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight,
toClose = append(toClose, a)
out.built, out.agent = true, s.agent
} else {
- built = &slot{agent: a, owns: true, leases: 1}
+ o := &own{router: true}
+ l.owners[a] = o
+ built = &slot{agent: a, own: o, leases: 1}
l.slots[key] = built
l.order = append(l.order, key)
out.evicted = l.evictLocked(&toClose)
@@ -384,9 +421,10 @@ func (r *Router) Load(ctx context.Context, name string) (*agent.Agent, error) {
if err != nil {
return nil, err
}
- // After release, so a hook that calls back in does not find this call holding a lease.
- rerr := r.lc.release(s)
- if herr := r.dispatchOutcome(key, out); herr != nil || rerr != nil {
+ // The lease is held across the hooks, so a concurrent eviction cannot close ctx.Agent under
+ // OnLoad; it is given back before Load returns.
+ herr := r.dispatchOutcome(key, out)
+ if rerr := r.lc.release(s); herr != nil || rerr != nil {
return nil, errors.Join(herr, rerr)
}
return s.agent, nil
@@ -414,6 +452,10 @@ func (r *Router) Attach(name string, a *agent.Agent) error {
return ErrClosed
}
prev := l.slots[key]
+ if o := l.owners[a]; o != nil { // built here, perhaps evicted but still leased: now the caller's
+ o.router = false
+ delete(l.owners, a)
+ }
l.slots[key] = &slot{agent: a}
if prev != nil && prev.agent != a { // the same agent changes hands, it is not closed
l.retire(prev, &toClose)
diff --git a/laya-go/router/lifecycle_review_test.go b/laya-go/router/lifecycle_review_test.go
new file mode 100644
index 00000000..7c23e104
--- /dev/null
+++ b/laya-go/router/lifecycle_review_test.go
@@ -0,0 +1,212 @@
+package router_test
+
+import (
+ "context"
+ "errors"
+ "math/rand"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/NandhaKishorM/laya/laya-go/agent"
+ "github.com/NandhaKishorM/laya/laya-go/router"
+)
+
+type panicSession struct{ probeSession }
+
+func (s *panicSession) Close() error { panic("session close exploded") }
+
+func TestPanicInEvictionCloseDoesNotWedgeTheRouter(t *testing.T) {
+ var evicted []string
+ var mu sync.Mutex
+ first := true
+ f := func(ctx context.Context, c router.Checkpoint) (*agent.Agent, error) {
+ if first {
+ first = false
+ return agent.Open(toyCheckpoint(), &panicSession{})
+ }
+ return newProbe(t, nil), nil
+ }
+ h := &router.HookFuncs{OnEvictFunc: func(c *router.EvictContext) { mu.Lock(); evicted = append(evicted, string(c.Model)); mu.Unlock() }}
+ r, err := router.New(router.WithAgentFactory(f), router.WithMaxLoaded(1), router.WithInstalledHooks(h))
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if _, err := r.Load(ctx, "english"); err != nil {
+ t.Fatal(err)
+ }
+ _, err = r.Lease(ctx, "multilingual") // evicts english, whose Close panics
+ if err == nil || !strings.Contains(err.Error(), "session close exploded") {
+ t.Fatalf("err = %v", err)
+ }
+ within(t, 2*time.Second, "Loaded after a panicking close", func() { r.Loaded() })
+ mu.Lock()
+ if len(evicted) != 1 || evicted[0] != "english" {
+ t.Fatalf("on_evict events %v: the panic skipped them", evicted)
+ }
+ mu.Unlock()
+ within(t, 2*time.Second, "next Load", func() {
+ if _, err := r.Load(ctx, "multilingual"); err != nil {
+ t.Error(err)
+ }
+ })
+}
+
+func TestPanicInOneCloseDoesNotSkipTheOthers(t *testing.T) {
+ good := &probeSession{}
+ n := 0
+ f := func(ctx context.Context, c router.Checkpoint) (*agent.Agent, error) {
+ n++
+ if n == 1 {
+ return agent.Open(toyCheckpoint(), &panicSession{})
+ }
+ return newProbe(t, good), nil
+ }
+ r, _ := router.New(router.WithAgentFactory(f), router.WithMaxLoaded(3))
+ ctx := context.Background()
+ r.Load(ctx, "english")
+ r.Load(ctx, "multilingual")
+ if _, err := r.UnloadAll(ctx); err == nil {
+ t.Fatal("the panic was lost")
+ }
+ if good.closes.Load() != 1 {
+ t.Fatal("a panicking close skipped the next agent")
+ }
+}
+
+func TestAttachOfAnEvictedLeasedAgentIsNeverClosedByTheRouter(t *testing.T) {
+ f := newFactory(t)
+ r := newRouter(t, f, router.WithMaxLoaded(1))
+ ls, err := r.Lease(context.Background(), "english")
+ if err != nil {
+ t.Fatal(err)
+ }
+ a := ls.Agent()
+ mustLoad(t, r, "multilingual") // evicts english while leased
+ if err := r.Attach("english", a); err != nil {
+ t.Fatal(err)
+ }
+ ls.Close()
+ if n := f.of(a).sess.closes.Load(); n != 0 {
+ t.Fatalf("an attached agent was closed by the Router (closes=%d)", n)
+ }
+}
+
+func TestLoadKeepsItsLeaseAcrossTheLoadHook(t *testing.T) {
+ f := newFactory(t)
+ r := newRouter(t, f, router.WithMaxLoaded(1))
+ var sawClosed atomic.Bool
+ var once sync.Once
+ r.AddHook(&router.HookFuncs{OnLoadFunc: func(c *router.LoadContext) {
+ if c.Model != router.English {
+ return
+ }
+ once.Do(func() {
+ go r.Load(context.Background(), "multilingual") // evicts english under the hook
+ b := f.of(c.Agent)
+ for i := 0; i < 100 && b.sess.closes.Load() == 0; i++ {
+ time.Sleep(5 * time.Millisecond)
+ }
+ sawClosed.Store(b.sess.closes.Load() > 0)
+ })
+ }})
+ mustLoad(t, r, "english")
+ if sawClosed.Load() {
+ t.Fatal("OnLoad held an agent that a concurrent eviction closed")
+ }
+}
+
+func TestFactoryNotCalledForABuildQueuedBehindClose(t *testing.T) {
+ f := newFactory(t)
+ f.entered = make(chan router.Checkpoint, 4)
+ r := newRouter(t, f)
+ gate := f.open(en)
+ go r.Load(context.Background(), "english")
+ <-f.entered
+ qerr := make(chan error, 1)
+ go func() { _, err := r.Load(context.Background(), "multilingual"); qerr <- err }() // queued on the token
+ time.Sleep(30 * time.Millisecond)
+ cdone := make(chan struct{})
+ go func() { r.Close(); close(cdone) }()
+ time.Sleep(30 * time.Millisecond)
+ close(gate)
+ <-cdone
+ if err := <-qerr; !errors.Is(err, router.ErrClosed) {
+ t.Fatalf("queued build: %v", err)
+ }
+ if f.calls() != 1 {
+ t.Fatalf("factory called %d times; a build started after Close", f.calls())
+ }
+}
+
+func TestBuildQueuedBehindTheTokenSeesAnAttach(t *testing.T) {
+ f := newFactory(t)
+ f.entered = make(chan router.Checkpoint, 4)
+ gate := f.open(en)
+ r := newRouter(t, f)
+ go r.Load(context.Background(), "english")
+ <-f.entered
+ res := make(chan *agent.Agent, 1)
+ go func() { a, _ := r.Load(context.Background(), "multilingual"); res <- a }()
+ time.Sleep(30 * time.Millisecond)
+ attached := newProbe(t, nil)
+ if err := r.Attach("multilingual", attached); err != nil {
+ t.Fatal(err)
+ }
+ close(gate)
+ if got := <-res; got != attached {
+ t.Fatal("the queued build did not use the attached agent")
+ }
+ if f.calls() != 1 {
+ t.Fatalf("factory called %d times for a checkpoint that was attached", f.calls())
+ }
+}
+
+func TestStressLeaseAccountingWithCancellation(t *testing.T) {
+ f := newFactory(t)
+ r := newRouter(t, f, router.WithMaxLoaded(1))
+ names := []string{"english", "multilingual", "typed-decisions"}
+ var wg sync.WaitGroup
+ for g := 0; g < 8; g++ {
+ wg.Add(1)
+ go func(seed int64) {
+ defer wg.Done()
+ rnd := rand.New(rand.NewSource(seed))
+ for i := 0; i < 400; i++ {
+ n := names[rnd.Intn(3)]
+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1+rnd.Intn(20))*time.Millisecond)
+ switch rnd.Intn(5) {
+ case 0, 1:
+ if l, err := r.Lease(ctx, n); err == nil {
+ if l.Agent() == nil || f.of(l.Agent()).sess.closes.Load() != 0 {
+ t.Error("lease handed out a closed agent")
+ }
+ l.Close()
+ l.Close()
+ }
+ case 2:
+ r.Load(ctx, n)
+ case 3:
+ r.Unload(ctx, n)
+ default:
+ r.UnloadAll(ctx)
+ }
+ cancel()
+ }
+ }(int64(g))
+ }
+ wg.Wait()
+ if err := r.Close(); err != nil {
+ t.Fatal(err)
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ for _, b := range f.made {
+ if c := b.sess.closes.Load(); c != 1 {
+ t.Errorf("%s closed %d times", b.label, c)
+ }
+ }
+}