Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

14.1 KB
Raw Blame
package sequence

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strings"

	"github.com/NandhaKishorM/laya/laya-go/internal/pyjson"
	"github.com/NandhaKishorM/laya/laya-go/internal/pyvalue"
	"github.com/NandhaKishorM/laya/laya-go/lang"
	"github.com/NandhaKishorM/laya/laya-go/schema"
	"github.com/NandhaKishorM/laya/laya-go/tokenizer"
)

// The head-budget literals of laya/common.py's build_head, which Python does not name.
const (
	// optionTokenCap is the tokens one option keeps after its [MASK], before any budget.
	optionTokenCap = 48
	// minOptionBudget is the head room the options must leave before they are re-capped.
	minOptionBudget = 16
	// minTokensPerOption is the floor on a re-capped option, its [MASK] included.
	minTokensPerOption = 4
	// minInstructionTokens is the floor on the instruction, "<type> question: " included.
	minInstructionTokens = 8
)

// ErrMissingSpecialToken is wrapped when the tokenizer names no CLS, SEP or mask token, which a
// sequence cannot be built without.
var ErrMissingSpecialToken = errors.New("tokenizer names no required special token")

// ErrUnsupportedState is wrapped when a state holds a value json.dumps would not write the same
// way: a struct, a channel, a non-string map key, invalid UTF-8.
var ErrUnsupportedState = errors.New("state cannot be serialized as laya serializes it")

// Budget is a checkpoint's sequence budget, from its rl_agent_config.json.
type Budget struct {
	// MaxLen is the whole sequence: 512 on english, 1024 on multilingual.
	MaxLen int
	// HeadMaxLen is the question half: 192 on english, 256 on multilingual.
	HeadMaxLen int
}

// DefaultWindow is the scan window when none is asked for: max(64, MaxLen - HeadMaxLen - 8).
func (b Budget) DefaultWindow() int {
	return max(64, b.MaxLen-b.HeadMaxLen-8)
}

// Keep says which end of an oversized state survives.
type Keep string

const (
	// KeepStart keeps the beginning of the state; the zero Keep means the same.
	KeepStart Keep = "start"
	// KeepEnd keeps the end: what laya does for a list state, a conversation's turns, whose
	// latest turn matters most.
	KeepEnd Keep = "end"
)

// State is a state tokenized once, to be paired with every question of a request.
type State struct {
	// IDs are the state's token ids, without special tokens.
	IDs []int32
	// Keep is the end that survives a cut.
	Keep Keep
}

// SerializeState is laya's serialize_state: a string passes through, anything else becomes
// json.dumps(state, ensure_ascii=False). nil is the text "null", as in Python. A Go map is
// written in sorted key order; use lang.Object (or ParseState) for an order-sensitive state. A
// json.Number is written as Python's json.loads and json.dumps would write the literal: an int
// literal stays an int of any size (up to Python's 4300 digits), and a float literal takes
// Python's float repr (1e+16, 100.0, 1e-05), with an overflow written as Infinity.
func SerializeState(state any) (string, error) {
	if s, ok := state.(string); ok {
		return s, nil
	}
	if _, isNumber := state.(json.Number); !isNumber {
		if rv := reflect.ValueOf(state); rv.IsValid() && rv.Kind() == reflect.String {
			return rv.String(), nil // a named string type is still a str
		}
	}
	text, err := pyvalue.Dumps(state)
	if err != nil {
		return "", fmt.Errorf("%w: %v", ErrUnsupportedState, err)
	}
	return text, nil
}

// ParseState reads a JSON document as Python's json.loads reads it, for use as a state: objects
// keep their key order (a repeated key keeps its first place and takes its last value) as
// lang.Object, and numbers stay json.Number, so that 2 serializes as 2 and 2.0 as 2.0. NaN,
// Infinity and -Infinity are accepted, as Python accepts them. A lone surrogate escape, a raw
// control character in a string, invalid UTF-8 and an int literal of more than 4300 digits are
// refused. data is the text json.loads would be given as a str, in UTF-8: a byte order mark is
// refused as Python refuses one in a str, and UTF-16 or UTF-32 input must be decoded first.
func ParseState(data []byte) (any, error) {
	v, err := pyvalue.Parse(data)
	if err != nil {
		return nil, fmt.Errorf("sequence: state: %w", err)
	}
	return public(v), nil
}

// public maps the decoder's pyjson.Object onto the exported lang.Object.
func public(v any) any {
	switch x := v.(type) {
	case pyjson.Object:
		out := make(lang.Object, len(x))
		for i, m := range x {
			out[i] = lang.Member{Key: m.Key.(string), Value: public(m.Value)}
		}
		return out
	case []any:
		out := make([]any, len(x))
		for i, item := range x {
			out[i] = public(item)
		}
		return out
	}
	return v
}

// EncodeState serializes state, scrubs the mask token from it and tokenizes it. Keep is KeepEnd
// for a slice or array state (not []byte) and KeepStart otherwise, as laya's predict decides.
func EncodeState(tok *tokenizer.Tokenizer, state any) (State, error) {
	sp, err := specials(tok)
	if err != nil {
		return State{}, err
	}
	text, err := SerializeState(state)
	if err != nil {
		return State{}, err
	}
	keep := KeepStart
	if isList(state) {
		keep = KeepEnd
	}
	return State{IDs: tok.Encode(strings.ReplaceAll(text, sp.maskToken, " ")), Keep: keep}, nil
}

// isList reports whether state is a Python list: a slice or array that is not bytes and not one
// of the ordered-object types, which are slices of members in Go but dicts in Python.
func isList(state any) bool {
	switch state.(type) {
	case lang.Object, pyjson.Object:
		return false
	}
	rv := reflect.ValueOf(state)
	if !rv.IsValid() || (rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array) {
		return false
	}
	return rv.Type().Elem().Kind() != reflect.Uint8
}

// special is the tokenizer's CLS, SEP and mask, resolved once per call.
type special struct {
	cls, sep, mask int32
	maskToken      string
}

func specials(tok *tokenizer.Tokenizer) (special, error) {
	if tok == nil {
		return special{}, errors.New("sequence: nil tokenizer")
	}
	var sp special
	var ok1, ok2, ok3 bool
	sp.cls, ok1 = tok.CLS()
	sp.sep, ok2 = tok.SEP()
	sp.mask, ok3 = tok.Mask()
	sp.maskToken = tok.MaskToken()
	switch {
	case !ok1:
		return special{}, fmt.Errorf("%w: no CLS token", ErrMissingSpecialToken)
	case !ok2:
		return special{}, fmt.Errorf("%w: no SEP token", ErrMissingSpecialToken)
	case !ok3 || sp.maskToken == "":
		return special{}, fmt.Errorf("%w: no mask token", ErrMissingSpecialToken)
	}
	return sp, nil
}

// Stats is what the head budget did to a question's options.
type Stats struct {
	// Options is how many options the question defines.
	Options int `json:"options"`
	// OptionsDistinct is how many still have a token span of their own. Fewer than Options
	// means the cap made two options the same tokens, which nothing downstream can see.
	OptionsDistinct int `json:"options_distinct"`
	// TokensPerOption is the cap applied to each option, or nil when none was.
	TokensPerOption *int `json:"tokens_per_option"`
}

// Collapsed reports that some options lost their distinct token span.
func (s Stats) Collapsed() bool { return s.OptionsDistinct < s.Options }

// Truncation is what the state budget dropped.
type Truncation struct {
	StateTokens        int  `json:"state_tokens"`
	StateTokensUsed    int  `json:"state_tokens_used"`
	StateTokensDropped int  `json:"state_tokens_dropped"`
	Truncated          bool `json:"truncated"`
}

// Head is the question half of a sequence, with no state.
type Head struct {
	// IDs run from [CLS] to the [SEP] closing the options.
	IDs []int32
	// Markers are the [MASK] positions, one per option slot, in presentation order.
	Markers []int
	Stats   Stats
}

// Sequence is one question paired with one state: one row of a batch.
type Sequence struct {
	// IDs are at most Budget.MaxLen token ids.
	IDs []int32
	// Markers are the [MASK] positions below MaxLen, one per option slot in presentation order:
	// slot s holds option q.OptionOrder()[s].
	Markers    []int
	Stats      Stats
	Truncation Truncation
}

// BuildHead is laya's build_head: [CLS] <type> question: <instructions> [SEP] [MASK] opt0 ...
// [SEP], with the options in q's OptionOrder.
func BuildHead(tok *tokenizer.Tokenizer, q schema.Question, headMaxLen int) (Head, error) {
	sp, err := specials(tok)
	if err != nil {
		return Head{}, err
	}
	if !q.Type().Valid() || q.NumOptions() == 0 {
		return Head{}, errors.New("sequence: the zero Question has no head; build one with schema.Choice, ChoiceOf, Score, Noul or NoulWith")
	}
	opts := q.RenderedOptions()
	order := q.OptionOrder()
	if order == nil {
		order = make([]int, len(opts))
		for i := range order {
			order[i] = i
		}
	}
	ins := strings.ReplaceAll(q.Instructions(), sp.maskToken, " ")
	headIDs := tok.Encode(string(q.Type()) + " question: " + ins)
	spans := make([][]int32, len(order))
	total := 0
	for slot, i := range order {
		// The cap is applied by the tokenizer, not after the fact, so a long description is
		// not tokenized in full to keep 48 tokens of it.
		text := " " + strings.ReplaceAll(opts[i], sp.maskToken, " ")
		span := append(make([]int32, 0, optionTokenCap+1), sp.mask)
		spans[slot] = append(span, tok.EncodeLimit(text, optionTokenCap)...)
		total += len(spans[slot])
	}
	budget := headMaxLen - total
	var perOption *int
	if budget < minOptionBudget {
		per := max(minTokensPerOption, floorDiv(headMaxLen-minOptionBudget, max(1, len(spans))))
		perOption = &per
		total = 0
		for slot, span := range spans {
			if len(span) > per {
				span = span[:per]
				spans[slot] = span
			}
			total += len(span)
		}
		budget = headMaxLen - total
	}
	if keep := max(minInstructionTokens, budget); keep < len(headIDs) {
		headIDs = headIDs[:keep]
	}
	ids := make([]int32, 0, 2+len(headIDs)+total+1)
	ids = append(ids, sp.cls)
	ids = append(ids, headIDs...)
	ids = append(ids, sp.sep)
	markers := make([]int, len(spans))
	distinct := make(map[string]struct{}, len(spans))
	for slot, span := range spans {
		markers[slot] = len(ids)
		ids = append(ids, span...)
		distinct[spanKey(span)] = struct{}{}
	}
	ids = append(ids, sp.sep)
	return Head{IDs: ids, Markers: markers, Stats: Stats{
		Options: len(spans), OptionsDistinct: len(distinct), TokensPerOption: perOption,
	}}, nil
}

// floorDiv is Python's a // b for b > 0.
func floorDiv(a, b int) int {
	q := a / b
	if a%b != 0 && a < 0 {
		q--
	}
	return q
}

// spanKey is a comparable key for a token span.
func spanKey(span []int32) string {
	b := make([]byte, 0, 4*len(span))
	for _, id := range span {
		b = append(b, byte(id), byte(id>>8), byte(id>>16), byte(id>>24))
	}
	return string(b)
}

func checkBudget(b Budget) error {
	if b.MaxLen <= 0 || b.HeadMaxLen <= 0 {
		return fmt.Errorf("sequence: budget needs a positive MaxLen and HeadMaxLen, got %d and %d", b.MaxLen, b.HeadMaxLen)
	}
	return nil
}

// MarkersError is returned by Build when the question's option markers do not all fit in
// MaxLen: the head is clamped to MaxLen, so the model would be asked about fewer options than the
// question defines. build_sequence drops them silently; the check that refuses is the agent's
// (Agent._encode_state), and Build carries it so no caller can forget it. ID is empty as Build
// returns it; set it to the question's id and Error is the agent's message, with its prefix.
type MarkersError struct {
	// ID is the question id, or "".
	ID string
	// Kept is how many markers fit below MaxLen, Total how many options the question defines.
	Kept, Total int
	// MaxLen and HeadMaxLen are the budget that was asked for.
	MaxLen, HeadMaxLen int
}

// Error is laya's message: "question 'id': only 3 of its 5 option markers fit in max_len=24 with
// head_max_len=16 spent on the question; lower head_max_len, raise max_len, or use fewer options".
func (e *MarkersError) Error() string {
	prefix := ""
	if e.ID != "" {
		prefix = "question " + pyjson.Repr(e.ID) + ": "
	}
	return fmt.Sprintf("%sonly %d of its %d option markers fit in max_len=%d with head_max_len=%d "+
		"spent on the question; lower head_max_len, raise max_len, or use fewer options",
		prefix, e.Kept, e.Total, e.MaxLen, e.HeadMaxLen)
}

// Build is laya's build_sequence: the head, then as much of state as the room after it allows,
// then [SEP], clamped to b.MaxLen. It is also the agent's marker check: when an option's marker
// falls past MaxLen it returns a *MarkersError instead of a sequence that asks about fewer
// options than were defined. Memory is linear in the question's total option text, as it is in
// Python: each option is tokenized to at most 48 tokens, but its rendered text is held whole.
func Build(tok *tokenizer.Tokenizer, q schema.Question, state State, b Budget) (Sequence, error) {
	if err := checkBudget(b); err != nil {
		return Sequence{}, err
	}
	head, err := BuildHead(tok, q, b.HeadMaxLen)
	if err != nil {
		return Sequence{}, err
	}
	sep, _ := tok.SEP() // BuildHead found it
	room := max(0, b.MaxLen-len(head.IDs)-1)
	// Not state[len-room:] by arithmetic on a zero room: that is the whole state in Python's
	// slice spelling, and none of it here.
	kept := state.IDs
	if len(kept) > room {
		if state.Keep == KeepEnd {
			kept = kept[len(kept)-room:]
		} else {
			kept = kept[:room]
		}
	}
	ids := make([]int32, 0, min(b.MaxLen, len(head.IDs)+len(kept)+1))
	ids = append(ids, head.IDs...)
	ids = append(ids, kept...)
	ids = append(ids, sep)
	if len(ids) > b.MaxLen {
		ids = ids[:b.MaxLen]
	}
	markers := make([]int, 0, len(head.Markers))
	for _, m := range head.Markers {
		if m < b.MaxLen {
			markers = append(markers, m)
		}
	}
	if len(markers) != len(head.Markers) {
		return Sequence{}, &MarkersError{Kept: len(markers), Total: len(head.Markers), MaxLen: b.MaxLen, HeadMaxLen: b.HeadMaxLen}
	}
	return Sequence{IDs: ids, Markers: markers, Stats: head.Stats, Truncation: Truncation{
		StateTokens:        len(state.IDs),
		StateTokensUsed:    len(kept),
		StateTokensDropped: len(state.IDs) - len(kept),
		Truncated:          len(kept) < len(state.IDs),
	}}, nil
}

// StateRoom is how many state tokens q leaves inside b, which is what Build keeps: laya's
// state_room. It is a property of the tokenizer as much as the question: one 88-option question
// leaves 150 tokens on english and 614 on multilingual.
func StateRoom(tok *tokenizer.Tokenizer, q schema.Question, b Budget) (int, error) {
	if err := checkBudget(b); err != nil {
		return 0, err
	}
	head, err := BuildHead(tok, q, b.HeadMaxLen)
	if err != nil {
		return 0, err
	}
	return max(0, b.MaxLen-len(head.IDs)-1), nil // -1 for the [SEP] that closes the state
}