Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
laya-go: hooks (OnPredictStart/End/Error), registries, dispatch with timeout and panic recovery
Adeshina Lasisib36c8a5a034d
diff --git a/laya-go/agent/agent.go b/laya-go/agent/agent.goindex c737ebcf..29010811 100644--- a/laya-go/agent/agent.go+++ b/laya-go/agent/agent.go@@ -64,14 +64,14 @@ type Agent struct {closed atomic.Bool // mirrors shut, for the check between chunksonce sync.Once // runs Close's bodycloseErr error // the session's Close error, read after once.Do++ hk *hookState // hooks; see hooks.go}-// Option adjusts an Agent at Open. No options exist yet; the parameter is here so adding one is-// not a breaking change.+// Option adjusts an Agent at Open: WithInstalledHooks, WithHookTimeout, WithHookErrors,+// WithConcurrentHooks and WithHookWarnings.type Option func(*options)-type options struct{}-// Open loads rl_agent_config.json and the tokenizer from checkpoint, a directory holding a// laya checkpoint, and takes ownership of s: on error it closes s, and otherwise Close does.// The tokenizer must name the pad, CLS, SEP and mask tokens. The session must run the graph@@ -80,20 +80,23 @@ func Open(checkpoint fs.FS, s infer.Session, opts ...Option) (*Agent, error) {if hasNilValue(s) {return nil, errors.New("agent: Open needs a session")}- var o options+ o := options{raise: true}for _, opt := range opts {if opt != nil {opt(&o)}}- a, err := open(checkpoint, s)+ if o.err != nil {+ return nil, errors.Join(o.err, s.Close())+ }+ a, err := open(checkpoint, s, o)if err != nil {return nil, errors.Join(err, s.Close())}return a, nil}-func open(checkpoint fs.FS, s infer.Session) (*Agent, error) {+func open(checkpoint fs.FS, s infer.Session, o options) (*Agent, error) {if checkpoint == nil {return nil, errors.New("agent: Open needs a checkpoint")}@@ -118,7 +121,7 @@ func open(checkpoint fs.FS, s infer.Session) (*Agent, error) {pad = id}}- return &Agent{tok: tok, padID: pad, cfg: cfg, temps: temps, sess: s}, nil+ return &Agent{tok: tok, padID: pad, cfg: cfg, temps: temps, sess: s, hk: newHookState(o)}, nil}// Config returns a copy of the checkpoint configuration the agent runs with.@@ -145,6 +148,7 @@ func (a *Agent) Close() error {a.shut = truea.closed.Store(true)a.mu.Unlock()+ a.hk.quit() // a call waiting on a hook gives up, so the wait below endsa.inflight.Wait()if a.sess != nil {a.closeErr = a.sess.Close()@@ -181,6 +185,15 @@ type call struct {// or a pyjson.Object, checked by checkMinConfidence.gated boolminConfidence any++ hooks []Hook // WithHooks, then the callables, in Python's order+ starts []Hook // WithOnPredictStart callables+ ends []Hook // WithOnPredictEnd callables+ skipDefault bool // WithoutDefaultHooks+ hooked bool // a hook option was given; PredictLong refuses it++ // maxLen and headMaxLen override the checkpoint's budgets when above 0; a start hook sets them.+ maxLen, headMaxLen int}// WithLanguage names the request's language as a BCP 47 tag. Its primary subtag selects a@@ -235,6 +248,9 @@ func (a *Agent) PredictBatch(ctx context.Context, states []any, qs schema.Questiopt(&c)}}+ if err := checkHooks(c.hooks); err != nil {+ return nil, err+ }// Validated up front, as laya does, so a bad threshold fails before any work.var g *gateif c.gated {@@ -250,6 +266,9 @@ func (a *Agent) PredictBatch(ctx context.Context, states []any, qs schema.Questireturn nil, err}defer a.leave()+ if hooks := a.composeHooks(c); len(hooks) > 0 {+ return a.predictHooked(ctx, states, qs, c, g, hooks)+ }preds, err := a.predictStates(ctx, states, qs, c)if err != nil {return nil, errdiff --git a/laya-go/agent/batch.go b/laya-go/agent/batch.goindex a698dfb8..f7954fa8 100644--- a/laya-go/agent/batch.go+++ b/laya-go/agent/batch.go@@ -53,7 +53,7 @@ func (a *Agent) predictChunks(ctx context.Context, states []any, qs schema.Questreturn nil, err}var err error- if encoded[i], err = a.encode(state, qs); err != nil {+ if encoded[i], err = a.encode(state, qs, c); err != nil {return nil, &StateError{Index: start + i, Err: err}}}@@ -91,12 +91,18 @@ func (a *Agent) predictChunks(ctx context.Context, states []any, qs schema.Quest}// encode is one state's question rows, tokenizing the state once for all of them.-func (a *Agent) encode(state any, qs schema.Questions) ([]sequence.Sequence, error) {+func (a *Agent) encode(state any, qs schema.Questions, c call) ([]sequence.Sequence, error) {st, err := sequence.EncodeState(a.tok, state)if err != nil {return nil, err}budget := sequence.Budget{MaxLen: a.cfg.MaxLen, HeadMaxLen: a.cfg.HeadMaxLen}+ if c.maxLen > 0 {+ budget.MaxLen = c.maxLen+ }+ if c.headMaxLen > 0 {+ budget.HeadMaxLen = c.headMaxLen+ }seqs := make([]sequence.Sequence, qs.Len())for i := range seqs {e := qs.At(i)diff --git a/laya-go/agent/hooks.go b/laya-go/agent/hooks.gonew file mode 100644index 00000000..76d1d36e--- /dev/null+++ b/laya-go/agent/hooks.go@@ -0,0 +1,507 @@+package agent++// Hooks are laya/hooks.py for the three events a single agent raises: OnPredictStart, OnPredictEnd+// and OnError. on_route, on_load and on_evict belong to the router and are not ported. Python's+// AsyncHook has no counterpart: a hook here is a plain method, and one that needs to wait runs+// its own goroutines (the agent itself waits for the method to return, bounded by+// WithHookTimeout).+//+// Composition order, as in Python: process-wide defaults, then the agent's installed hooks, then+// the call's WithHooks, then its WithOnPredictStart and WithOnPredictEnd callables.+// WithoutDefaultHooks drops only the first group.+//+// Unlike Python's ONNXAgent, whose hooks_concurrent defaults to true, hooks run serially unless+// WithConcurrentHooks is given: one hook runs at a time across all calls of an agent, so a hook+// needs no locking of its own. A hook must not call back into the agent that is running it while+// the hooks are serial; the call would wait for itself.+//+// A hook that panics is recovered and counts as a failed hook, as a raised exception does in+// Python. With WithHookErrors(true), the default, the first failure ends the chain and fails the+// call; with false it is reported to the WithHookWarnings sink and the chain goes on.+//+// WithHookTimeout bounds the wait for each hook. Go cannot stop a goroutine, so a hook that+// overruns keeps running after the call has returned an error: the timeout protects the request,+// not the process. Such a hook must not touch the PredictContext it was given (the call has moved+// on and owns it), and it no longer holds the serial lock. Close does not wait for a running+// hook: a call that is waiting on one gives up with ErrClosed.++import (+ "context"+ "crypto/rand"+ "encoding/hex"+ "fmt"+ "log"+ "math"+ "reflect"+ "strconv"+ "sync"+ "time"++ "github.com/NandhaKishorM/laya/laya-go/schema"+)++// Hook observes one Predict or PredictBatch call. A hook that wants only some events embeds+// BaseHook, or uses HookFuncs.+type Hook interface {+ // OnPredictStart runs before inference. It may rewrite ctx.States, ctx.Questions,+ // ctx.MaxLen and ctx.HeadMaxLen, or answer the call with ctx.Skip.+ OnPredictStart(ctx *PredictContext)+ // OnPredictEnd runs once the call is over, on success and on failure. ctx.Err is set on+ // failure. It may rewrite ctx.Results.+ OnPredictEnd(ctx *PredictContext)+ // OnError runs when the call failed, before OnPredictEnd, with ctx.Err set.+ OnError(ctx *PredictContext)+}++// BaseHook implements Hook with no-ops, for embedding.+type BaseHook struct{}++func (BaseHook) OnPredictStart(*PredictContext) {}+func (BaseHook) OnPredictEnd(*PredictContext) {}+func (BaseHook) OnError(*PredictContext) {}++// HookFuncs is a Hook made of functions; a nil one is a no-op. Use it as *HookFuncs: a value+// holds funcs, which cannot be compared, so RemoveHook could never find it. The error function+// is OnErr because a field cannot share its name with the OnError method.+type HookFuncs struct {+ OnStart, OnEnd, OnErr func(*PredictContext)+}++func (f *HookFuncs) OnPredictStart(c *PredictContext) { callIf(f.OnStart, c) }+func (f *HookFuncs) OnPredictEnd(c *PredictContext) { callIf(f.OnEnd, c) }+func (f *HookFuncs) OnError(c *PredictContext) { callIf(f.OnErr, c) }++func callIf(fn func(*PredictContext), c *PredictContext) {+ if fn != nil {+ fn(c)+ }+}++// HookUsage is the token count of a whole call: the sum over its predictions.+type HookUsage struct {+ InputTokens, OutputTokens int+}++// PredictContext is what a hook sees of one call. Hooks of one call share it and run one after+// another, so a rewrite made by one is seen by the next.+type PredictContext struct {+ States []any+ Questions schema.Questions+ // RunID is 32 hex characters, shared by every hook of the call.+ RunID string+ // Results is nil until the call has answered (or a hook has skipped it); an empty non-nil+ // slice is a real answer to a call with no states.+ Results []schema.Prediction+ Model string+ Agent *Agent+ // MaxLen and HeadMaxLen override the checkpoint's budgets for this call when above 0.+ MaxLen, HeadMaxLen int+ // Usage is summed over Results; set before OnPredictEnd.+ Usage HookUsage+ StartedAt time.Time+ // ElapsedMS is set before OnPredictEnd, not before OnError.+ ElapsedMS float64+ // Err is the failure, set before OnError.+ Err error+}++// Skip answers the call without inference: one result for the whole call, or one per state in+// ctx.States. It does not stop the chain, and the last Skip wins. Any other count is an error,+// returned so the hook can panic with it or give up.+func (c *PredictContext) Skip(results []schema.Prediction) error {+ if len(results) != 1 && len(results) != len(c.States) {+ return fmt.Errorf("ctx.skip() takes one result for the whole call or one per state in ctx.states (%d); got %d", len(c.States), len(results))+ }+ c.Results = results+ return nil+}++// aggregateUsage sums the per-state usage blocks.+func aggregateUsage(results []schema.Prediction) HookUsage {+ var u HookUsage+ for _, r := range results {+ u.InputTokens += r.Usage.InputTokens+ u.OutputTokens += r.Usage.OutputTokens+ }+ return u+}++// --- options ---++type options struct {+ installed []Hook+ timeout time.Duration+ raise bool+ concurrent bool+ warn func(string)+ err error+}++// WithInstalledHooks installs hooks on the agent, as AddHook does after Open.+func WithInstalledHooks(hooks ...Hook) Option {+ return func(o *options) {+ if err := checkHooks(hooks); err != nil && o.err == nil {+ o.err = err+ }+ o.installed = append(o.installed, hooks...)+ }+}++// WithHookTimeout bounds the wait for each hook. d must be positive; leave the option out for no+// limit.+func WithHookTimeout(d time.Duration) Option {+ return func(o *options) {+ if err := validateTimeout(d.Seconds()); err != nil && o.err == nil {+ o.err = err+ }+ o.timeout = d+ }+}++// WithHookErrors chooses what a failing hook does: raise true (the default) fails the call,+// false warns and goes on.+func WithHookErrors(raise bool) Option { return func(o *options) { o.raise = raise } }++// WithConcurrentHooks lets hooks of concurrent calls run at the same time; the default runs one+// at a time.+func WithConcurrentHooks() Option { return func(o *options) { o.concurrent = true } }++// WithHookWarnings sets where a hook failure that does not fail the call is reported; the+// default is the log package.+func WithHookWarnings(sink func(msg string)) Option { return func(o *options) { o.warn = sink } }++// validateTimeout is hooks.validate_timeout: positive and finite.+func validateTimeout(seconds float64) error {+ if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds <= 0 {+ return fmt.Errorf("hooks_timeout must be a positive finite number or None; got %s", strconv.FormatFloat(seconds, 'g', -1, 64))+ }+ return nil+}++// WithHooks adds hooks to one call, after the defaults and the installed ones.+func WithHooks(hooks ...Hook) CallOption {+ return func(c *call) {+ c.hooked = true+ c.hooks = append(c.hooks, hooks...)+ }+}++// WithOnPredictStart adds a function run as an OnPredictStart hook, after the WithHooks ones.+func WithOnPredictStart(fns ...func(*PredictContext)) CallOption {+ return func(c *call) {+ c.hooked = true+ for _, fn := range fns {+ c.starts = append(c.starts, &HookFuncs{OnStart: fn})+ }+ }+}++// WithOnPredictEnd adds a function run as an OnPredictEnd hook, after every WithOnPredictStart.+func WithOnPredictEnd(fns ...func(*PredictContext)) CallOption {+ return func(c *call) {+ c.hooked = true+ for _, fn := range fns {+ c.ends = append(c.ends, &HookFuncs{OnEnd: fn})+ }+ }+}++// WithoutDefaultHooks leaves the process-wide defaults out of this call.+func WithoutDefaultHooks() CallOption {+ return func(c *call) {+ c.hooked = true+ c.skipDefault = true+ }+}++// checkHooks refuses a nil hook, Python's "entries must be instances" for the one mistake the+// type system leaves open.+func checkHooks(hooks []Hook) error {+ for i, h := range hooks {+ if hasNilValue(h) {+ return fmt.Errorf("agent: hook %d is nil", i)+ }+ }+ return nil+}++// --- registries ---++var defaults struct {+ mu sync.RWMutex+ hooks []Hook+}++// SetDefaultHooks replaces the process-wide hooks every agent runs first.+func SetDefaultHooks(hooks ...Hook) error {+ if err := checkHooks(hooks); err != nil {+ return err+ }+ defaults.mu.Lock()+ defaults.hooks = append([]Hook(nil), hooks...)+ defaults.mu.Unlock()+ return nil+}++// AddDefaultHook appends to the process-wide hooks.+func AddDefaultHook(hooks ...Hook) error {+ if err := checkHooks(hooks); err != nil {+ return err+ }+ defaults.mu.Lock()+ defaults.hooks = append(defaults.hooks, hooks...)+ defaults.mu.Unlock()+ return nil+}++// ClearDefaultHooks removes every process-wide hook.+func ClearDefaultHooks() {+ defaults.mu.Lock()+ defaults.hooks = nil+ defaults.mu.Unlock()+}++// DefaultHooks returns a copy of the process-wide hooks.+func DefaultHooks() []Hook {+ defaults.mu.RLock()+ defer defaults.mu.RUnlock()+ return append([]Hook(nil), defaults.hooks...)+}++// hookState is the agent's hook registry and dispatch settings.+type hookState struct {+ mu sync.RWMutex+ installed []Hook+ timeout time.Duration+ raise bool+ serial bool+ warn func(string)+ sem chan struct{} // the serial lock, a channel so a waiter can give up+ done chan struct{} // closed by Close+ closeOnce sync.Once+}++func newHookState(o options) *hookState {+ h := &hookState{installed: append([]Hook(nil), o.installed...), timeout: o.timeout, raise: o.raise,+ serial: !o.concurrent, warn: o.warn, sem: make(chan struct{}, 1), done: make(chan struct{})}+ if h.warn == nil {+ h.warn = func(msg string) { log.Print(msg) }+ }+ return h+}++func (h *hookState) quit() {+ if h == nil {+ return+ }+ h.closeOnce.Do(func() { close(h.done) })+}++// AddHook installs hooks on the agent, after those already there. A hook added twice runs twice.+func (a *Agent) AddHook(hooks ...Hook) error {+ if err := checkHooks(hooks); err != nil {+ return err+ }+ a.hk.mu.Lock()+ a.hk.installed = append(a.hk.installed, hooks...)+ a.hk.mu.Unlock()+ return nil+}++// RemoveHook removes every installed copy of hook, compared by identity, and reports whether+// there was one. A hook whose dynamic type cannot be compared (a struct holding a func, used as+// a value) is never found; pass a pointer.+func (a *Agent) RemoveHook(hook Hook) bool {+ a.hk.mu.Lock()+ defer a.hk.mu.Unlock()+ kept := a.hk.installed[:0:0]+ for _, h := range a.hk.installed {+ if !sameHook(h, hook) {+ kept = append(kept, h)+ }+ }+ removed := len(kept) != len(a.hk.installed)+ a.hk.installed = kept+ return removed+}++// InstallHooks installs hooks until the returned function is called, which removes one copy of+// each (the most recent) and leaves everything else, including what was added meanwhile.+// Overlapping installs may be undone in either order.+func (a *Agent) InstallHooks(hooks ...Hook) (undo func(), err error) {+ if err := a.AddHook(hooks...); err != nil {+ return nil, err+ }+ var once sync.Once+ return func() {+ once.Do(func() {+ a.hk.mu.Lock()+ defer a.hk.mu.Unlock()+ for _, h := range hooks {+ for i := len(a.hk.installed) - 1; i >= 0; i-- {+ if sameHook(a.hk.installed[i], h) {+ a.hk.installed = append(a.hk.installed[:i:i], a.hk.installed[i+1:]...)+ break+ }+ }+ }+ })+ }, nil+}++// Hooks returns a copy of the installed hooks.+func (a *Agent) Hooks() []Hook {+ a.hk.mu.RLock()+ defer a.hk.mu.RUnlock()+ return append([]Hook(nil), a.hk.installed...)+}++func sameHook(a, b Hook) bool {+ ta, tb := reflect.TypeOf(a), reflect.TypeOf(b)+ return ta == tb && ta.Comparable() && a == b+}++// composeHooks is the hooks of one call, in order.+func (a *Agent) composeHooks(c call) []Hook {+ var hooks []Hook+ if !c.skipDefault {+ hooks = DefaultHooks()+ }+ a.hk.mu.RLock()+ hooks = append(hooks, a.hk.installed...)+ a.hk.mu.RUnlock()+ hooks = append(hooks, c.hooks...)+ hooks = append(hooks, c.starts...)+ return append(hooks, c.ends...)+}++// --- dispatch ---++type hookEvent struct {+ name string // Python's event name+ call func(Hook, *PredictContext)+}++var (+ evStart = hookEvent{"on_predict_start", Hook.OnPredictStart}+ evEnd = hookEvent{"on_predict_end", Hook.OnPredictEnd}+ evError = hookEvent{"on_error", Hook.OnError}+)++func hookName(h Hook) string {+ t := reflect.TypeOf(h)+ for t.Kind() == reflect.Pointer {+ t = t.Elem()+ }+ return t.Name()+}++// dispatch runs one event over the hooks in order, as hooks.dispatch.+func (a *Agent) dispatch(ctx context.Context, ev hookEvent, hooks []Hook, pc *PredictContext) error {+ for _, h := range hooks {+ err := a.callHook(ctx, ev, h, pc)+ if err == nil {+ continue+ }+ if a.hk.raise {+ return err+ }+ a.hk.warn(fmt.Sprintf("laya: hook %s.%s failed: %s", hookName(h), ev.name, err))+ }+ return nil+}++// callHook runs one hook in its own goroutine, so a timeout, Close or the caller's context can+// stop the wait. Under the serial lock the lock covers the wait, not the goroutine, as in Python.+func (a *Agent) callHook(ctx context.Context, ev hookEvent, h Hook, pc *PredictContext) error {+ hk := a.hk+ if hk.serial {+ select {+ case hk.sem <- struct{}{}:+ defer func() { <-hk.sem }()+ case <-hk.done:+ return ErrClosed+ case <-ctx.Done():+ return ctx.Err()+ }+ }+ res := make(chan error, 1) // buffered: an abandoned hook still finishes without blocking+ go func() {+ defer func() {+ if r := recover(); r != nil {+ if err, ok := r.(error); ok {+ res <- err+ } else {+ res <- fmt.Errorf("%v", r)+ }+ }+ }()+ ev.call(h, pc)+ res <- nil+ }()+ var expired <-chan time.Time+ if hk.timeout > 0 {+ t := time.NewTimer(hk.timeout)+ defer t.Stop()+ expired = t.C+ }+ select {+ case err := <-res:+ return err+ case <-expired:+ return fmt.Errorf("laya: hook %s exceeded %gs", hookName(h), hk.timeout.Seconds())+ case <-hk.done:+ return ErrClosed+ case <-ctx.Done():+ return ctx.Err()+ }+}++// attached is an error that keeps the message and identity of the first failure and carries a+// second one along, as Python's __context__ does: a failing OnError or OnPredictEnd never masks+// the error that caused it.+type attached struct{ first, second error }++func (e *attached) Error() string { return e.first.Error() }+func (e *attached) Unwrap() []error { return []error{e.first, e.second} }++func newRunID() string {+ var b [16]byte+ _, _ = rand.Read(b[:])+ return hex.EncodeToString(b[:])+}++// predictHooked is PredictBatch with hooks, as ONNXAgent.predict_batch: start, inference unless a+// hook answered, then on any failure OnError, and always OnPredictEnd.+func (a *Agent) predictHooked(ctx context.Context, states []any, qs schema.Questions, c call, g *gate, hooks []Hook) ([]schema.Prediction, error) {+ pc := &PredictContext{States: states, Questions: qs, RunID: newRunID(), Model: schema.Model, Agent: a, StartedAt: time.Now()}+ err := a.dispatch(ctx, evStart, hooks, pc)+ if err == nil && pc.Results == nil {+ c.maxLen, c.headMaxLen = pc.MaxLen, pc.HeadMaxLen+ pc.Results, err = a.predictStates(ctx, pc.States, pc.Questions, c)+ }+ if err != nil {+ pc.Err = err+ if err2 := a.dispatch(ctx, evError, hooks, pc); err2 != nil {+ err = &attached{err, err2}+ }+ }+ pc.ElapsedMS = float64(time.Since(pc.StartedAt)) / float64(time.Millisecond)+ if pc.Results != nil {+ pc.Usage = aggregateUsage(pc.Results)+ if g != nil {+ g.apply(pc.Results)+ }+ }+ if err2 := a.dispatch(ctx, evEnd, hooks, pc); err2 != nil {+ if err == nil {+ err = err2+ } else {+ err = &attached{err, err2}+ }+ }+ if err != nil {+ return nil, err+ }+ return pc.Results, nil+}diff --git a/laya-go/agent/long.go b/laya-go/agent/long.goindex 011d7653..c6c3e07b 100644--- a/laya-go/agent/long.go+++ b/laya-go/agent/long.go@@ -50,7 +50,8 @@ type LongPrediction struct {// state. Errors are those of PredictBatch, and sequence.ErrNoRoom and sequence.ErrStrideGap when// the questions leave no room or the stride leaves gaps. WithMinConfidence and// WithMinConfidenceMap are refused with an error before any work, as Python raises TypeError for-// min_confidence; hooks are not applied, and Go has no calibration.+// min_confidence; WithHooks, WithOnPredictStart and WithOnPredictEnd are refused the same way: PredictLong runs no hooks, as in+// Python, and Go has no calibration.func (a *Agent) PredictLong(ctx context.Context, state any, qs schema.Questions, opts ...CallOption) (LongPrediction, error) {var c callfor _, opt := range opts {@@ -63,6 +64,9 @@ func (a *Agent) PredictLong(ctx context.Context, state any, qs schema.Questions,// would return ungated answers to a caller who asked for a gate.return LongPrediction{}, errors.New("agent: PredictLong takes no confidence gate; use Predict or PredictBatch with WithMinConfidence")}+ if c.hooked {+ return LongPrediction{}, errors.New("agent: PredictLong runs no hooks; use Predict or PredictBatch with WithHooks")+ }if err := ctx.Err(); err != nil {return LongPrediction{}, err}