Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

18.5 KB
Raw Blame
package router

// The model lifecycle is laya/router.py's load, attach, preload, unload and eviction, with one
// addition Python does not need: a Lease. Python frees an agent by dropping its reference and lets
// the garbage collector decide when; an Agent here owns a native session that Close frees, so
// eviction must not close an agent a caller is still predicting with. A slot counts its leases,
// and an agent the Router built closes when it has left the Router and its last lease is
// released.
//
// Locking, as in Python: mu guards the slots, the LRU order, the in-flight builds and the flags.
// It is never held while a factory runs, while an agent closes, or while a hook runs. build
// serialises factory calls across checkpoints, so two cold loads never hold two checkpoints in
// flight at once. A load of a resident checkpoint never touches build.

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/NandhaKishorM/laya/laya-go/agent"
)

// DefaultMaxLoaded is how many checkpoints a Router keeps resident unless WithMaxLoaded says
// otherwise.
const DefaultMaxLoaded = 2

// ErrClosed is returned by a Router that has been closed.
var ErrClosed = errors.New("router: closed")

// AgentFactory builds the agent for a checkpoint. It owns where the weights come from, the
// device and the agent options. It is called at most once at a time per Router and never with a
// lock held, and ctx is the context of the call that triggered the build. The Router closes the
// agent it returns, so a factory must return a fresh agent on every call, never one it shares. A
// factory must not call Load, Lease, Preload, Unload, UnloadAll or Close on the Router that is
// calling it: it holds the build token, so a call that builds, or waits for a build, waits for
// itself.
type AgentFactory func(ctx context.Context, c Checkpoint) (*agent.Agent, error)

// WithAgentFactory lets the Router load checkpoints. Without it Route works and every load fails.
func WithAgentFactory(f AgentFactory) Option {
	return func(c *config) error { c.factory = f; return nil }
}

// WithMaxLoaded sets how many checkpoints stay resident before the least recently used is
// evicted; n must be at least 1. Attach and Preload raise it to fit what they make resident,
// and never lower it.
func WithMaxLoaded(n int) Option {
	return func(c *config) error {
		if n < 1 {
			return fmt.Errorf("router: max loaded must be at least 1, got %d", n)
		}
		c.maxLoaded = n
		return nil
	}
}

type slot struct {
	agent   *agent.Agent
	leases  int
	retired bool // left the Router
	own     *own // non-nil when the Router built the agent; Attach clears it
}

// own is the Router's claim to close an agent it built. It is shared by every slot of that agent,
// so attaching the agent hands it to the caller even while an earlier, retired slot is leased.
type own struct{ router bool }

func (s *slot) owns() bool { return s.own != nil && s.own.router }

type inflight struct {
	done chan struct{}
	err  error // set before done closes
}

type lifecycle struct {
	factory AgentFactory
	build   chan struct{} // one token: serialises factory calls

	mu      sync.Mutex
	slots   map[Checkpoint]*slot
	order   []Checkpoint // least recently used first
	loading map[Checkpoint]*inflight
	max     int
	closed  bool
	owners  map[*agent.Agent]*own // agents the Router built and has not closed
}

func (l *lifecycle) init(f AgentFactory, max int) {
	l.factory, l.max = f, max
	l.build = make(chan struct{}, 1)
	l.slots = map[Checkpoint]*slot{}
	l.loading = map[Checkpoint]*inflight{}
	l.owners = map[*agent.Agent]*own{}
}

// outcome is what one acquire did, so its hooks fire after the locks are released.
type outcome struct {
	built    bool
	agent    *agent.Agent
	evicted  []Checkpoint
	closeErr error // closing what the build pushed out failed; the build still happened
}

// touch moves key to the most recently used end. mu is held.
func (l *lifecycle) touch(key Checkpoint) {
	for i, k := range l.order {
		if k == key {
			l.order = append(l.order[:i], l.order[i+1:]...)
			break
		}
	}
	l.order = append(l.order, key)
}

// retire marks s as gone from the Router and collects its agent for closing when nothing is using
// it and the Router built it. mu is held.
func (l *lifecycle) retire(s *slot, toClose *[]*agent.Agent) {
	s.retired = true
	if s.leases <= 0 && s.owns() {
		delete(l.owners, s.agent)
		*toClose = append(*toClose, s.agent)
	}
}

// evictLocked drops least recently used slots until max holds and returns who left. mu is held.
func (l *lifecycle) evictLocked(toClose *[]*agent.Agent) []Checkpoint {
	var evicted []Checkpoint
	for len(l.order) > l.max {
		victim := l.order[0]
		l.order = l.order[1:]
		if s := l.slots[victim]; s != nil {
			delete(l.slots, victim)
			evicted = append(evicted, victim)
			l.retire(s, toClose)
		}
	}
	return evicted
}

// safeClose closes a, turning a panic in its session's Close into an error so one bad agent
// cannot skip the others or the events that follow.
func safeClose(a *agent.Agent) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("router: closing an agent panicked: %v", r)
		}
	}()
	return a.Close()
}

func closeAll(agents []*agent.Agent) error {
	var errs []error
	for _, a := range agents {
		if err := safeClose(a); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Join(errs...)
}

// release gives back one lease and closes the agent when it has left and this was the last.
func (l *lifecycle) release(s *slot) error {
	var toClose *agent.Agent
	l.mu.Lock()
	s.leases--
	if s.leases <= 0 && s.retired && s.owns() {
		toClose = s.agent
		delete(l.owners, s.agent)
	}
	l.mu.Unlock()
	if toClose != nil {
		return safeClose(toClose)
	}
	return nil
}

func isContextErr(err error) bool {
	return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
}

// acquire returns the slot for key with one lease taken, building it if needed. Concurrent
// callers for one key share a single build; a caller whose build failed returns the error and
// caches nothing, and the others get the same error. A waiter does not inherit the builder's
// cancellation: it builds for itself.
func (l *lifecycle) acquire(ctx context.Context, key Checkpoint, out *outcome) (*slot, error) {
	for {
		l.mu.Lock()
		if l.closed {
			l.mu.Unlock()
			return nil, ErrClosed
		}
		if s := l.slots[key]; s != nil { // not behind build: a resident load never waits for a cold one
			l.touch(key)
			s.leases++
			l.mu.Unlock()
			return s, nil
		}
		wait := l.loading[key]
		var mine *inflight
		if wait == nil {
			mine = &inflight{done: make(chan struct{})}
			l.loading[key] = mine
		}
		l.mu.Unlock()
		if mine != nil {
			return l.doBuild(ctx, key, mine, out)
		}
		select {
		case <-wait.done:
		case <-ctx.Done():
			return nil, ctx.Err()
		}
		if wait.err != nil {
			if isContextErr(wait.err) && ctx.Err() == nil {
				continue
			}
			return nil, fmt.Errorf("router: loading %s failed on the call that was building it: %w", key, wait.err)
		}
	}
}

func (l *lifecycle) create(ctx context.Context, key Checkpoint) (*agent.Agent, error) {
	if l.factory == nil {
		return nil, errors.New("router: this Router can route but not load: no agent factory was given; use WithAgentFactory")
	}
	a, err := l.factory(ctx, key)
	if err != nil {
		return nil, fmt.Errorf("router: cannot load the %s checkpoint: %w", key, err)
	}
	if a == nil {
		return nil, fmt.Errorf("router: the agent factory returned no agent for %s", key)
	}
	return a, nil
}

// doBuild is the build this caller owns: mine is the handle the others wait on.
func (l *lifecycle) doBuild(ctx context.Context, key Checkpoint, mine *inflight, out *outcome) (built *slot, err error) {
	var toClose []*agent.Agent
	finish := func() { // mu is held; runs once: nothing after it can panic (closes are safeClose)
		if l.loading[key] == mine {
			delete(l.loading, key)
		}
		close(mine.done)
	}
	defer func() {
		rec := recover()
		if rec != nil {
			err = fmt.Errorf("router: building %s panicked: %v", key, rec)
		}
		if err != nil {
			built = nil
			l.mu.Lock()
			mine.err = err
			finish()
			l.mu.Unlock()
		}
		if rec != nil {
			panic(rec)
		}
	}()
	select {
	case l.build <- struct{}{}:
	case <-ctx.Done():
		return nil, ctx.Err()
	}
	released := false
	releaseBuild := func() {
		if !released {
			released = true
			<-l.build
		}
	}
	defer releaseBuild()

	l.mu.Lock()
	if l.closed { // Close came while this caller waited for the token: do not start a build
		l.mu.Unlock()
		return nil, ErrClosed
	}
	if s := l.slots[key]; s != nil { // built or attached while this caller waited for the build token
		l.touch(key)
		s.leases++
		finish()
		l.mu.Unlock()
		return s, nil
	}
	l.mu.Unlock()

	a, err := l.create(ctx, key)
	if err != nil {
		return nil, err
	}

	l.mu.Lock()
	if s := l.slots[key]; s != nil { // attached while it was building: keep that one
		l.touch(key)
		s.leases++
		built = s
		toClose = append(toClose, a)
		out.built, out.agent = true, s.agent
	} else {
		o := &own{router: true}
		l.owners[a] = o
		built = &slot{agent: a, own: o, leases: 1}
		l.slots[key] = built
		l.order = append(l.order, key)
		out.evicted = l.evictLocked(&toClose)
		out.built, out.agent = true, a
	}
	finish()
	l.mu.Unlock()
	releaseBuild()
	// Outside every lock: closing a native session can be slow.
	out.closeErr = closeAll(toClose)
	return built, nil
}

// take acquires a lease for key and settles what the build owes: a failed close of an evicted
// agent fails the call, but the events are still delivered.
func (r *Router) take(ctx context.Context, key Checkpoint) (*slot, *outcome, error) {
	out := &outcome{}
	s, err := r.lc.acquire(ctx, key, out)
	if err != nil {
		return nil, nil, err
	}
	if out.closeErr != nil {
		err := out.closeErr
		if rerr := r.lc.release(s); rerr != nil {
			err = errors.Join(err, rerr)
		}
		if herr := r.dispatchOutcome(key, out); herr != nil {
			err = errors.Join(err, herr)
		}
		return nil, nil, err
	}
	return s, out, nil
}

// dispatchOutcome fires the evictions a build caused, then its load, stopping at the first hook
// error as Python does. It must be called with no lock held.
func (r *Router) dispatchOutcome(key Checkpoint, out *outcome) error {
	for _, ev := range out.evicted {
		if err := r.dispatchEvict(ev); err != nil {
			return err
		}
	}
	if out.built {
		return r.dispatchLoad(key, out.agent)
	}
	return nil
}

// Lease is a borrowed agent. The agent stays open until Close is called, however many times the
// Router evicts or unloads it meanwhile. Close it when done; the zero Lease holds nothing.
type Lease struct {
	r        *Router
	s        *slot
	key      Checkpoint
	released atomic.Bool
}

// Checkpoint is the checkpoint the lease was taken for.
func (l *Lease) Checkpoint() Checkpoint { return l.key }

// Agent is the leased agent, or nil once the lease is closed.
func (l *Lease) Agent() *agent.Agent {
	if l == nil || l.s == nil || l.released.Load() {
		return nil
	}
	return l.s.agent
}

// Close gives the lease back; when the agent has left the Router and this was its last lease, the
// agent closes and Close returns its error. Closing twice is safe: only the first counts.
func (l *Lease) Close() error {
	if l == nil || l.s == nil || !l.released.CompareAndSwap(false, true) {
		return nil
	}
	return l.r.lc.release(l.s)
}

// Lease borrows the agent for a checkpoint (a name or alias), building it on first use. OnEvict
// and OnLoad hooks have run when it returns, with the lease already held, so the agent is never
// closed under them. If a hook fails the lease is given back and the error returned, and the
// checkpoint stays resident.
func (r *Router) Lease(ctx context.Context, name string) (*Lease, error) {
	key, err := NormaliseName(name)
	if err != nil {
		return nil, err
	}
	s, out, err := r.take(ctx, key)
	if err != nil {
		return nil, err
	}
	if err := r.dispatchOutcome(key, out); err != nil {
		return nil, errors.Join(err, r.lc.release(s))
	}
	return &Lease{r: r, s: s, key: key}, nil
}

// Load returns the agent for a checkpoint, building it on first use. Concurrent callers share one
// build. The agent is valid only while it stays resident: eviction and Unload close an agent the
// Router built, so a reference held across either is a closed session. Use Lease, which holds the
// agent open, unless the caller manages the lifetime itself. If an OnLoad hook unloads, or closes
// the Router, for the same checkpoint, Load returns an agent that is already closed and no error. If a hook fails Load returns the
// error and no agent, and the checkpoint stays resident.
func (r *Router) Load(ctx context.Context, name string) (*agent.Agent, error) {
	key, err := NormaliseName(name)
	if err != nil {
		return nil, err
	}
	s, out, err := r.take(ctx, key)
	if err != nil {
		return nil, err
	}
	// The lease is held across the hooks, so a concurrent eviction cannot close ctx.Agent under
	// OnLoad; it is given back before Load returns.
	herr := r.dispatchOutcome(key, out)
	if rerr := r.lc.release(s); herr != nil || rerr != nil {
		return nil, errors.Join(herr, rerr)
	}
	return s.agent, nil
}

// Attach makes an agent the caller built resident under a checkpoint name, instead of loading a
// second copy. The Router never closes an attached agent: the caller keeps ownership. It does not
// evict; MaxLoaded is raised to fit everything resident. An agent already resident under the name
// is replaced, and closes when its last lease is released if the Router built it. Attaching the
// agent already there hands it to the caller: the Router stops owning it and never closes it. Python registers an unknown name on the spot; the
// names here are fixed, so an unknown one is an *UnknownModelError. No hook runs.
func (r *Router) Attach(name string, a *agent.Agent) error {
	key, err := NormaliseName(name)
	if err != nil {
		return err
	}
	if a == nil {
		return errors.New("router: Attach needs an agent")
	}
	l := &r.lc
	var toClose []*agent.Agent
	l.mu.Lock()
	if l.closed {
		l.mu.Unlock()
		return ErrClosed
	}
	prev := l.slots[key]
	if o := l.owners[a]; o != nil { // built here, perhaps evicted but still leased: now the caller's
		o.router = false
		delete(l.owners, a)
	}
	l.slots[key] = &slot{agent: a}
	if prev != nil && prev.agent != a { // the same agent changes hands, it is not closed
		l.retire(prev, &toClose)
	}
	l.touch(key)
	if len(l.slots) > l.max {
		l.max = len(l.slots)
	}
	l.mu.Unlock()
	return closeAll(toClose)
}

// Preload builds checkpoints up front so no request pays a cold load; with no names, all of them.
// MaxLoaded is raised to fit both the checkpoints asked for and everything resident, so staged
// preloading evicts neither. A resident checkpoint is left where it is in the LRU order. It stops
// at the first failure.
func (r *Router) Preload(ctx context.Context, names ...string) error {
	keys := Checkpoints()
	if len(names) > 0 {
		keys = keys[:0:0]
		for _, n := range names {
			key, err := NormaliseName(n)
			if err != nil {
				return err
			}
			keys = append(keys, key)
		}
	}
	l := &r.lc
	l.mu.Lock()
	if l.closed {
		l.mu.Unlock()
		return ErrClosed
	}
	union := map[Checkpoint]bool{}
	for _, k := range keys {
		union[k] = true
	}
	for k := range l.slots {
		union[k] = true
	}
	if len(union) > l.max {
		l.max = len(union)
	}
	l.mu.Unlock()
	for _, key := range keys {
		l.mu.Lock()
		already := l.slots[key] != nil
		l.mu.Unlock()
		if already {
			continue
		}
		s, out, err := r.take(ctx, key)
		if err != nil {
			return err
		}
		herr := r.dispatchOutcome(key, out) // the lease is held across the hooks, as in Load
		if rerr := l.release(s); herr != nil || rerr != nil {
			return errors.Join(herr, rerr)
		}
	}
	return nil
}

// evictAll fires OnEvict for each freed checkpoint, stopping at the first hook error, after the
// agents that could close have. The events are owed whether or not a close failed.
func (r *Router) evictAll(freed []Checkpoint, toClose []*agent.Agent) error {
	cerr := closeAll(toClose)
	for _, k := range freed {
		if err := r.dispatchEvict(k); err != nil {
			return errors.Join(cerr, err)
		}
	}
	return cerr
}

// Unload frees one checkpoint (a name or alias) and returns it if it was resident. It waits for
// an in-flight build of that checkpoint only. The slot goes at once; its agent closes when the
// last lease is released. OnEvict fires for a checkpoint freed.
func (r *Router) Unload(ctx context.Context, name string) ([]Checkpoint, error) {
	key, err := NormaliseName(name)
	if err != nil {
		return nil, err
	}
	l := &r.lc
	for {
		var toClose []*agent.Agent
		var freed []Checkpoint
		l.mu.Lock()
		wait := l.loading[key]
		if wait == nil {
			if s := l.slots[key]; s != nil {
				delete(l.slots, key)
				freed = append(freed, key)
				l.retire(s, &toClose)
			}
			l.removeOrder(key)
		}
		l.mu.Unlock()
		if wait == nil {
			return freed, r.evictAll(freed, toClose)
		}
		select {
		case <-wait.done:
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}
}

// removeOrder drops key from the LRU order. mu is held.
func (l *lifecycle) removeOrder(key Checkpoint) {
	for i, k := range l.order {
		if k == key {
			l.order = append(l.order[:i], l.order[i+1:]...)
			return
		}
	}
}

// UnloadAll frees every checkpoint, least recently used first, after any build in flight has
// finished, and returns who was freed.
func (r *Router) UnloadAll(ctx context.Context) ([]Checkpoint, error) {
	l := &r.lc
	for {
		var toClose []*agent.Agent
		var freed []Checkpoint
		var waits []*inflight
		l.mu.Lock()
		for _, w := range l.loading {
			waits = append(waits, w)
		}
		if len(waits) == 0 {
			freed = append(freed, l.order...)
			for _, k := range l.order {
				if s := l.slots[k]; s != nil {
					l.retire(s, &toClose)
				}
			}
			l.slots = map[Checkpoint]*slot{}
			l.order = nil
		}
		l.mu.Unlock()
		if len(waits) == 0 {
			return freed, r.evictAll(freed, toClose)
		}
		for _, w := range waits {
			select {
			case <-w.done:
			case <-ctx.Done():
				return nil, ctx.Err()
			}
		}
	}
}

// Loaded lists the resident checkpoints, least recently used first.
func (r *Router) Loaded() []Checkpoint {
	r.lc.mu.Lock()
	defer r.lc.mu.Unlock()
	return append([]Checkpoint(nil), r.lc.order...)
}

// MaxLoaded is how many checkpoints the Router keeps before it evicts.
func (r *Router) MaxLoaded() int {
	r.lc.mu.Lock()
	defer r.lc.mu.Unlock()
	return r.lc.max
}

// Close seals the Router and frees every checkpoint. A build in flight finishes first, and its
// caller gets the lease it asked for; that agent closes when the lease is released. An agent
// under lease closes at its last release, an attached agent is left to its owner, and Close is
// idempotent. Loads after Close return ErrClosed; Route still works.
func (r *Router) Close() error {
	r.lc.mu.Lock()
	r.lc.closed = true
	r.lc.mu.Unlock()
	_, err := r.UnloadAll(context.Background())
	return err
}