Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

11.6 KB
Raw Blame
package agent

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/NandhaKishorM/laya/laya-go/infer"
	"github.com/NandhaKishorM/laya/laya-go/internal/decode"
	"github.com/NandhaKishorM/laya/laya-go/schema"
	"github.com/NandhaKishorM/laya/laya-go/sequence"
	"github.com/NandhaKishorM/laya/laya-go/tokenizer"
)

// ErrClosed is returned by an Agent that has been closed. A session closed underneath an open
// agent yields a wrapped infer.ErrClosed instead.
var ErrClosed = errors.New("agent: agent is closed")

// StateError identifies the state at fault in a PredictBatch call.
type StateError struct {
	// Index is the state's position in the states slice.
	Index int
	Err   error
}

// Error names the state and the cause.
func (e *StateError) Error() string {
	if e.Err == nil {
		return fmt.Sprintf("agent: state %d", e.Index)
	}
	// The cause is a complete message ("agent: state is nil; ..."); keep one prefix and one subject.
	msg := e.Err.Error()
	if rest, ok := strings.CutPrefix(msg, "agent: state "); ok {
		if _, nested := e.Err.(*StateError); !nested {
			msg = rest
		}
	} else {
		msg = strings.TrimPrefix(msg, "agent: ")
	}
	return fmt.Sprintf("agent: state %d: %s", e.Index, msg)
}

// Unwrap returns the cause.
func (e *StateError) Unwrap() error { return e.Err }

// Agent answers typed questions about a state with a laya checkpoint, through an infer.Session.
// It is the port of laya's ONNXAgent, and it is safe for concurrent use. Use Open: the zero
// Agent is closed.
type Agent struct {
	tok   *tokenizer.Tokenizer
	padID int32
	cfg   Config
	temps *decode.Temperatures
	sess  infer.Session

	mu       sync.Mutex     // guards shut, so a call enters or is refused without waiting
	shut     bool           // set by Close under mu
	inflight sync.WaitGroup // calls that entered
	closed   atomic.Bool    // mirrors shut, for the check between chunks
	once     sync.Once      // runs Close's body
	closeErr error          // the session's Close error, read after once.Do

	hk *hookState // hooks; see hooks.go
}

// Option adjusts an Agent at Open: WithInstalledHooks, WithHookTimeout, WithHookErrors,
// WithConcurrentHooks, WithHookWarnings, WithLangTemperatures, WithExpectedSHA256 and WithCalibration.
type Option func(*options)

// Open loads rl_agent_config.json and the tokenizer from checkpoint, a directory holding a
// laya checkpoint, and takes ownership of s: on error it closes s, and otherwise Close does.
// The tokenizer must name the pad, CLS, SEP and mask tokens. The session must run the graph
// exported from that same checkpoint; nothing here can tell.
func Open(checkpoint fs.FS, s infer.Session, opts ...Option) (*Agent, error) {
	if hasNilValue(s) {
		return nil, errors.New("agent: Open needs a session")
	}
	o := options{raise: true}
	for _, opt := range opts {
		if opt != nil {
			opt(&o)
		}
	}
	if o.err != nil {
		return nil, errors.Join(o.err, s.Close())
	}
	a, err := open(checkpoint, s, o)
	if err != nil {
		return nil, errors.Join(err, s.Close())
	}
	return a, nil
}

func open(checkpoint fs.FS, s infer.Session, o options) (*Agent, error) {
	if checkpoint == nil {
		return nil, errors.New("agent: Open needs a checkpoint")
	}
	want, err := expectedDigests(o.sha256)
	if err != nil {
		return nil, err
	}
	if err := verifyDigests(checkpoint, want); err != nil {
		return nil, err
	}
	cfg, temps, err := loadConfig(checkpoint, o.langTemps)
	if err != nil {
		return nil, err
	}
	if len(o.calibration) > 0 {
		if temps, err = applyCalibration(&cfg, temps, o.calibration); err != nil {
			return nil, &ConfigError{Field: calibrationField, Err: err}
		}
	}
	tok, err := tokenizer.LoadFS(checkpoint, ".")
	if err != nil {
		return nil, &ConfigError{Field: tokenizerField, Err: err}
	}
	var pad int32
	for _, t := range []struct {
		name string
		id   func() (int32, bool)
	}{{"pad", tok.Pad}, {"CLS", tok.CLS}, {"SEP", tok.SEP}, {"mask", tok.Mask}} {
		id, ok := t.id()
		if !ok {
			return nil, fmt.Errorf("agent: %w: no %s token", sequence.ErrMissingSpecialToken, t.name)
		}
		if t.name == "pad" {
			pad = id
		}
	}
	return &Agent{tok: tok, padID: pad, cfg: cfg, temps: temps, sess: s, hk: newHookState(o)}, nil
}

// Config returns a copy of the checkpoint configuration the agent runs with.
func (a *Agent) Config() Config {
	cfg := a.cfg
	cfg.Warnings = append([]string(nil), cfg.Warnings...)
	return cfg
}

// Close marks the agent closed, waits for calls in flight and closes the session. Close blocks
// until calls in flight return. Cancelling a call's context ends it between chunks and releases a
// call still queued for the serial hook lock, but a hook that has started is always waited for:
// only WithHookTimeout bounds that wait. A call that has not yet entered returns ErrClosed without
// waiting. Close is idempotent: a second call, concurrent or not, waits for the first and returns
// the same error. Close must not be called from inside a call, from the session's Run or from a
// hook: it would wait for that call, which waits for Close (with a hook timeout set, a hook's
// Close returns only after the timeout releases its call).
func (a *Agent) Close() error {
	a.once.Do(func() {
		defer func() {
			if r := recover(); r != nil { // once is spent either way: later calls report this
				a.closeErr = fmt.Errorf("agent: the session's Close panicked: %v", r)
				panic(r)
			}
		}()
		a.mu.Lock()
		a.shut = true
		a.closed.Store(true)
		a.mu.Unlock()
		a.hk.quit() // releases calls queued for the serial lock before OnPredictStart
		a.inflight.Wait()
		if a.sess != nil {
			a.closeErr = a.sess.Close()
		}
	})
	return a.closeErr
}

// enter registers a call, or reports ErrClosed. It never waits for a call in flight. Every
// successful enter is paired with leave.
func (a *Agent) enter() error {
	a.mu.Lock()
	defer a.mu.Unlock()
	if a.shut || a.sess == nil {
		return ErrClosed
	}
	a.inflight.Add(1)
	return nil
}

func (a *Agent) leave() { a.inflight.Done() }

// CallOption adjusts one Predict or PredictBatch call.
type CallOption func(*call)

type call struct {
	lang   string
	batch  int
	sort   bool
	window int
	stride int

	// gated is set by WithMinConfidence or WithMinConfidenceMap; minConfidence is then a float64
	// or a pyjson.Object, checked by checkMinConfidence.
	gated         bool
	minConfidence any

	hooks       []Hook // WithHooks, then the callables, in Python's order
	starts      []Hook // WithOnPredictStart callables
	ends        []Hook // WithOnPredictEnd callables
	skipDefault bool   // WithoutDefaultHooks
	hooked      bool   // a hook option was given; PredictLong refuses it

	// maxLen and headMaxLen override the checkpoint's budgets when above 0; a start hook sets them.
	maxLen, headMaxLen int
}

// WithLanguage names the request's language as a BCP 47 tag. Its primary subtag selects a
// language's temperatures when the agent has any; with none configured it changes nothing.
func WithLanguage(tag string) CallOption { return func(c *call) { c.lang = tag } }

// WithBatchSize caps the states sent per session run; 0 or less means all in one run. A run
// holds states x questions rows.
func WithBatchSize(states int) CallOption { return func(c *call) { c.batch = states } }

// WithSortByLength(true) pads less: states are ordered by length inside a lookahead of eight
// batches before they are chunked. It needs a batch size above 1 and below the number of states,
// and results stay in input order. A different batch shape can move a probability by a rounding
// step. WithSortByLength(false) is the default.
func WithSortByLength(on bool) CallOption { return func(c *call) { c.sort = on } }

// Predict answers every question about one state. Its errors are those of PredictBatch, except
// that a failing state's error is returned without the *StateError around it.
func (a *Agent) Predict(ctx context.Context, state any, qs schema.Questions, opts ...CallOption) (schema.Prediction, error) {
	preds, err := a.PredictBatch(ctx, []any{state}, qs, opts...)
	if err != nil {
		if se, ok := err.(*StateError); ok { // the top-level error only, never one inside a RunError
			err = se.Err
		}
		return schema.Prediction{}, err
	}
	return preds[0], nil
}

// PredictBatch answers the questions for each state; results are in input order. A state is a
// string, or any value sequence.SerializeState accepts: a slice is a conversation whose end
// survives truncation. A nil state or nil pointer is refused; a nil slice or map is an empty
// list or dict. With no questions each state gets an empty answer and zero usage, with no model
// run.
//
// WithMinConfidence or WithMinConfidenceMap gates every answer: it sets low_confidence,
// abstention and abstention_threshold as laya does. An invalid threshold is refused first, as a
// *MinConfidenceError, before any state is encoded or run.
//
// Errors may be wrapped; match them with errors.Is and errors.As:
//   - *MinConfidenceError, for an invalid threshold, before anything else is checked;
//   - ErrClosed, once Close has been called (a context already done is reported first);
//   - ctx.Err(), checked on entry, before each state is encoded and between runs;
//   - *StateError, naming the state, around sequence.ErrUnsupportedState (also for a nil
//     state), sequence.ErrMissingSpecialToken or a *sequence.MarkersError with its ID set;
//   - *RunError, around whatever the session returned (so errors.Is finds context.Canceled and
//     infer.ErrClosed through it), a *infer.ShapeError, or a logit that cannot be decoded.
func (a *Agent) PredictBatch(ctx context.Context, states []any, qs schema.Questions, opts ...CallOption) ([]schema.Prediction, error) {
	var c call
	for _, opt := range opts {
		if opt != nil {
			opt(&c)
		}
	}
	if err := checkHooks(c.hooks); err != nil {
		return nil, err
	}
	// Validated up front, as laya does, so a bad threshold fails before any work.
	var g *gate
	if c.gated {
		var err error
		if g, err = checkMinConfidence(c.minConfidence); err != nil {
			return nil, err
		}
	}
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	if err := a.enter(); err != nil {
		return nil, err
	}
	defer a.leave()
	if hooks := a.composeHooks(c); len(hooks) > 0 {
		return a.predictHooked(ctx, states, qs, c, g, hooks)
	}
	preds, err := a.inferStates(ctx, states, qs, c)
	if err != nil {
		return nil, err
	}
	if g != nil {
		g.apply(preds)
	}
	return preds, nil
}

// predictStates is PredictBatch past the entry checks, for a caller that has already entered.
func (a *Agent) predictStates(ctx context.Context, states []any, qs schema.Questions, c call) ([]schema.Prediction, error) {
	for i, state := range states {
		if isNil(state) {
			return nil, &StateError{Index: i, Err: fmt.Errorf("agent: state is nil; pass a string, a map or a slice: %w", sequence.ErrUnsupportedState)}
		}
	}
	if len(states) == 0 {
		return []schema.Prediction{}, nil
	}
	if qs.Len() == 0 {
		preds := make([]schema.Prediction, len(states))
		for i := range preds {
			preds[i] = schema.Prediction{Model: schema.Model, Answers: schema.Answers{},
				Usage: schema.Usage{TruncatedQuestions: []string{}}}
		}
		return preds, nil
	}
	return a.predictChunks(ctx, states, qs, c)
}

// isNil reports an untyped nil or a nil pointer; a nil slice or map is an empty list or dict.
func isNil(state any) bool {
	if state == nil {
		return true
	}
	rv := reflect.ValueOf(state)
	return rv.Kind() == reflect.Pointer && rv.IsNil()
}

// hasNilValue reports an untyped nil or an interface holding a nil pointer, map, slice, func or
// chan, which a method call would trip over.
func hasNilValue(v any) bool {
	if v == nil {
		return true
	}
	switch rv := reflect.ValueOf(v); rv.Kind() {
	case reflect.Pointer, reflect.Interface, reflect.Map, reflect.Slice, reflect.Func, reflect.Chan:
		return rv.IsNil()
	}
	return false
}