Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #569 from AlKor13/feat/report-option-collapse
Nandakishor1921330a5d08
diff --cc laya/common.pyindex 15a4cbe2,08b67db3..404d994c--- a/laya/common.py+++ b/laya/common.py@@@ -144,61 -150,38 +151,90 @@@ def build_sequence# not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of itst = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]ids = ids + st + [tok.sep_token_id]- return ids[:max_len], [m for m in markers if m < max_len]+ ids, markers = ids[:max_len], [m for m in markers if m < max_len]+ if not return_stats:+ return ids, markers+ # Two options that share a prefix can come out of the cut as the same token span: the marker+ # count still matches the option count, so the guard in `Agent._encode_state` passes and+ # nothing downstream can tell that the question lost the ability to name them apart. Counted+ # on the capped option ids, before assembly: re-slicing the finished sequence cannot close+ # the last option's span -- it runs on into the serialized state, which differs per request,+ # so the last option always looks distinguishable however it collided (#538).+ return ids, markers, {+ "options": len(opt_ids),+ "options_distinct": len({tuple(o) for o in opt_ids}),+ "tokens_per_option": per_option,+ }+++ def collapsed_options(qids, items) -> Dict[str, Dict[str, Optional[int]]]:+ """The questions whose options no longer have a token span each, from per-item stats.++ `total` is the number of options the question defines, not the number of markers that+ reached the sequence: a report counted from the markers would say "43/58" about a request+ where 28 options never made it into the input at all.+ """+ out = {}+ for qid, item in zip(qids, items):+ stats = item.get("options")+ if stats and stats["options_distinct"] < stats["options"]:+ out[qid] = {"total": stats["options"], "distinct": stats["options_distinct"],+ "tokens_per_option": stats["tokens_per_option"]}+ return out+class _DynamicMultiheadAttention(nn.MultiheadAttention):+ """`nn.MultiheadAttention` that keeps the shapes it traces.++ The stock module reshapes the packed projection with sizes captured while tracing, so under+ the legacy ONNX exporter the traced sequence length becomes a constant and a model exported+ from a short dummy input only runs at that length. The parameters and the maths are the same+ here; the reshape uses only constant shape arguments (`chunk` / `unflatten` / `flatten`) and+ `scaled_dot_product_attention`, the kernel the stock path already uses when weights are not+ requested.+ """++ def forward(self, query, key, value, key_padding_mask=None, need_weights=True,+ attn_mask=None, average_attn_weights=True, is_causal=False):+ if (need_weights or self.in_proj_weight is None or self.bias_k is not None+ or self.bias_v is not None+ or (attn_mask is not None and attn_mask.dtype != torch.bool)):+ # Weight averaging, additive masks and the optional k/v bias are not on the traced+ # path; the stock implementation keeps them correct.+ return super().forward(query, key, value, key_padding_mask=key_padding_mask,+ need_weights=need_weights, attn_mask=attn_mask,+ average_attn_weights=average_attn_weights, is_causal=is_causal)+ if self.batch_first:+ query, key, value = query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)+ # (T, B, E) from here, matching the stock module's internals; attention runs on (B, H, T, D).+ if query is key is value:+ q, k, v = (part.unflatten(-1, (self.num_heads, self.head_dim)).permute(1, 2, 0, 3)+ for part in F.linear(query, self.in_proj_weight, self.in_proj_bias).chunk(3, dim=-1))+ else:+ embed_dim = query.shape[-1]+ wq, wk, wv = self.in_proj_weight.split(embed_dim, dim=0)+ bq, bk, bv = ((None, None, None) if self.in_proj_bias is None+ else self.in_proj_bias.split(embed_dim, dim=0))+ q, k, v = (+ F.linear(t, w, b).unflatten(-1, (self.num_heads, self.head_dim)).permute(1, 2, 0, 3)+ for t, w, b in ((query, wq, bq), (key, wk, bk), (value, wv, bv))+ )+ mask = None+ if attn_mask is not None:+ mask = ~attn_mask+ if key_padding_mask is not None:+ # `== 0` keeps this correct for a bool mask and for the 0 / -inf float mask the encoder+ # layer hands over (`F._canonical_mask`), where `~` would not be defined.+ keep = key_padding_mask[:, None, None, :] == 0+ mask = keep if mask is None else mask & keep+ attn = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=is_causal and mask is None,+ dropout_p=self.dropout if self.training else 0.0)+ attn = attn.permute(2, 0, 1, 3).flatten(-2)+ if self.batch_first:+ attn = attn.transpose(0, 1)+ return self.out_proj(attn), None++class DecisionModel(nn.Module):"""Bidirectional transformer encoder backbone + typed decision head."""diff --cc laya/onnx_agent.pyindex 327dd07a,88ef3355..79939e51--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@@ -478,36 -298,41 +479,36 @@@ class ONNXAgent(HookRegistry)add_special_tokens=False,)["input_ids"]+ items = []for qid in ids:- q = self._to_internal(questions[qid])+ q = internal[qid]- seq, markers = build_sequence(+ seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,- truncate_left=truncate_left, state_ids=state_ids,+ truncate_left=truncate_left, state_ids=state_ids, return_stats=True,)- if len(markers) != len(render_options(q)):- raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))+ n_opts = len(render_options(q))+ if len(markers) != n_opts:+ # The markers are placed at absolute positions and `build_sequence` then drops the+ # ones past `max_len`, so this is about the question fitting in the sequence --+ # `head_max_len` is how much of it the options were given, and `max_len` is the+ # ceiling that dropped them. Naming only `head_max_len` pointed at the wrong knob+ # in both directions: lowering it shortens the option block and can make the+ # call succeed, while raising it makes the overflow worse.+ raise ValueError(+ "question %r: its %d options and question text need %d tokens, more than "+ "max_len=%d allows once head_max_len=%d is spent on them; lower head_max_len, "+ "raise max_len, or use fewer options"+ % (qid, n_opts, len(seq), max_len, head_max_len))- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})+ return items- b = collate_items([items], self.tok.pad_token_id)-- # Prepare ONNX inputs as numpy arrays- ort_inputs = {- "input_ids": b["input_ids"].numpy().astype(np.int64),- "attention_mask": b["attention_mask"].numpy().astype(np.int64),- "marker_pos": b["marker_pos"].numpy().astype(np.int64),- "marker_mask": b["marker_mask"].numpy().astype(bool),- "qtype": b["qtype"].numpy().astype(np.int64),- }-- # Run ONNX inference- ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)- logits = ort_outs[0]- act_logits = ort_outs[1]-- # Compute softmax for actions manually in numpy- act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))- act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)-+ def _decode_answers(self, logits, act, items: List[Dict[str, Any]], ids: List[str],+ internal: Dict[str, Any], offset: int,+ lang: Optional[str] = None) -> Dict[str, Any]:+ """Decode the `len(ids)` head rows starting at `offset` into this state's answers."""answers = {}- n_tokens = int(b["attention_mask"].sum())-for r, qid in enumerate(ids):- q = self._to_internal(questions[qid])+ q = internal[qid]k = len(items[r]["markers"])qt = QTYPES[q["t"]]t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])@@@ -554,81 -379,16 +555,86 @@@"answer_confidence": ans_conf,"action": ext,}+ return answers++ def _infer_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],+ max_len: Optional[int] = None, head_max_len: Optional[int] = None,+ lang: Optional[str] = None,+ batch_size: Optional[int] = None,+ sort_by_length: bool = False) -> List[Dict[str, Any]]:+ """Validate once, then encode, collate, run and decode in chunks of `batch_size` states.++ With `sort_by_length`, chunks are formed from length-sorted states inside a lookahead+ window of eight batches (the same contract as `Agent.predict_batch`), so a batch pads to+ a shorter maximum; results are returned in input order either way.+ """+ from .agent import Agent as _Agent++ ids = list(questions.keys())+ if not ids:+ return [{"model": "laya-rl-agent-onnx", "answers": {},+ "usage": {"input_tokens": 0, "output_tokens": 0}} for _ in states]+ for qid in ids:+ _Agent._check_question(qid, questions[qid])+ internal = {qid: self._to_internal(questions[qid]) for qid in ids}+ max_len = self.cfg.get("max_len", 512) if max_len is None else max_len+ head_max_len = self.cfg.get("head_max_len", 192) if head_max_len is None else head_max_len+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)++ results: List[Dict[str, Any]] = []+ reorder = sort_by_length and 1 < chunk < len(states)+ # Bound the tokenized lookahead independently of the input size (same as the torch Agent):+ # sort within windows of eight batches, using each state's longest encoded question row.+ window = chunk * 8 if reorder else chunk+ for start in range(0, len(states), window):+ part = states[start:start + window]+ encoded = [self._encode_state(st, ids, internal, max_len, head_max_len) for st in part]+ order = list(range(len(encoded)))+ if reorder:+ order.sort(key=lambda i: max(len(item["ids"]) for item in encoded[i]))+ window_results: List[Optional[Dict[str, Any]]] = [None] * len(encoded)+ for offset in range(0, len(order), chunk):+ indices = order[offset:offset + chunk]+ per_state_items = [encoded[i] for i in indices]++ b = collate_items(per_state_items, self.tok.pad_token_id)++ # Prepare ONNX inputs as numpy arrays+ ort_inputs = {+ "input_ids": b["input_ids"].numpy().astype(np.int64),+ "attention_mask": b["attention_mask"].numpy().astype(np.int64),+ "marker_pos": b["marker_pos"].numpy().astype(np.int64),+ "marker_mask": b["marker_mask"].numpy().astype(bool),+ "qtype": b["qtype"].numpy().astype(np.int64),+ }- usage = {"input_tokens": n_tokens, "output_tokens": 0}- collapsed = collapsed_options(ids, items)- if collapsed:- usage["options"] = collapsed- return {- "model": "laya-rl-agent-onnx",- "answers": answers,- "usage": usage,- }+ # Run ONNX inference+ ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)+ logits = ort_outs[0]+ act_logits = ort_outs[1]++ # Compute softmax for actions manually in numpy+ act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))+ act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)+ att = b["attention_mask"]++ row = 0+ for index, items in zip(indices, per_state_items):+ nrows = len(items)+ n_tokens = int(att[row:row + nrows].sum())++ usage = {"input_tokens": n_tokens, "output_tokens": 0}++ # Only when a question lost options to the head budget, as on the torch Agent.++ collapsed = collapsed_options(ids, items)++ if collapsed:++ usage["options"] = collapsed+ window_results[index] = {+ "model": "laya-rl-agent-onnx",+ "answers": self._decode_answers(logits, act, items, ids, internal, row, lang=lang),- "usage": {"input_tokens": n_tokens, "output_tokens": 0},++ "usage": usage,+ }+ row += nrows+ results.extend(window_results)+ return resultsdef decide(self, state: Union[str, dict, list], schema: Any = None, *,questions: Optional[Dict[str, Dict[str, Any]]] = None, return_details: bool = False,diff --cc tests/test_hooks_api.pyindex 59b7a193,9d8f0b67..4e8a38ab--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@@ -163,96 -157,21 +163,111 @@@ for label, cls in (("Agent", Agent), ("for method in ("add_hook", "remove_hook", "hooks_installed"):check_true("%s/%s exists" % (label, method), callable(getattr(cls, method, None)))+# The LangChain runnables batch: laya.integrations.langchain's own suite checks what+# batch() returns, so these lines pin only the caller-visible shape. A rename, or losing+# the keyword-only return_exceptions, would break LCEL and LangGraph map-reduce silently.+from laya.integrations.langchain import ( # noqa: E402+ LayaEvaluator,+ LayaGuardrail,+ LayaRouter,+ LayaTriage,+)++for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):+ for method in ("invoke", "batch", "abatch"):+ check_true("langchain/%s.%s exists" % (label, method),+ callable(getattr(cls, method, None)))+ check("langchain/%s/batch defaults" % label,+ [(p.name, p.kind.name, p.default) for p in sig(cls.batch).values()],+ [("self", "POSITIONAL_OR_KEYWORD", inspect.Parameter.empty),+ ("inputs", "POSITIONAL_OR_KEYWORD", inspect.Parameter.empty),+ ("config", "POSITIONAL_OR_KEYWORD", None),+ ("return_exceptions", "KEYWORD_ONLY", False),+ ("kwargs", "VAR_KEYWORD", inspect.Parameter.empty)])+ check("langchain/%s/abatch defaults" % label,+ [(p.name, p.kind.name, p.default) for p in sig(cls.abatch).values()],+ [(p.name, p.kind.name, p.default) for p in sig(cls.batch).values()])++# --------------------------------------------------------------- LangChain decision node+# `LayaDecision` is the LCEL form of `laya.decide`, so its constructor has to keep accepting the+# same schema argument and runner overrides. The module imports without langchain-core installed+# (the runnable base falls back to plain object), so these checks hold in every CI lane.+from laya.integrations.langchain import LayaDecision # noqa: E402++for param, default in (("return_details", False), ("state_key", None), ("agent", None),+ ("base_url", None), ("api_key", None), ("model", None)):+ check_param("LayaDecision.__init__", LayaDecision.__init__, param, default)+check_param("LayaDecision.__init__", LayaDecision.__init__, "decision_schema",+ inspect.Parameter.empty, inspect.Parameter.POSITIONAL_OR_KEYWORD)+check_true("LayaDecision/invoke exists", callable(getattr(LayaDecision, "invoke", None)))+check_true("LayaDecision/__all__", "LayaDecision" in laya.__all__)+check_true("LayaDecision/laya attribute", hasattr(laya, "LayaDecision"))+check_true("LayaDecision/integrations export",+ "LayaDecision" in __import__("laya.integrations", fromlist=["__all__"]).__all__)+++# --------------------------------------------------------------- LangChain token budget+# `max_len`/`head_max_len` are the per-request knobs that decide how many tokens each option of a+# choice question gets. Every runnable has to carry both, under the names the core API uses, or a+# chain has no way to widen a question that overflows the checkpoint's budget.+from laya.integrations.langchain import ( # noqa: E402+ LayaEvaluator,+ LayaGuardrail,+ LayaRouter,+ LayaTriage,+)++for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):+ # Without langchain-core the runnables are plain objects, so there is no model schema to+ # carry the field; only the constructor contract applies in that lane.+ declared = getattr(cls, "model_fields", None)+ for param in ("max_len", "head_max_len"):+ check_param("%s.__init__" % label, cls.__init__, param, None)+ if declared is not None:+ check_true("%s/%s declared field" % (label, param), param in declared)+++# The LangChain runnables have to carry the per-call hook arguments too, or a chain cannot+# install the cache/guard patterns from docs/hooks/patterns.md on a single node. They default to+# None (meaning "inherit the runner"), which is what keeps an unset node byte-identical to today.+# `hooks_concurrent` is deliberately absent: like the core predict surfaces, it is not per-call.+from laya.integrations.langchain import ( # noqa: E402+ LayaEvaluator,+ LayaGuardrail,+ LayaRouter,+ LayaTriage,+)++PER_CALL_HOOK_PARAMS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise",+ "hooks_timeout")++for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):+ for param in PER_CALL_HOOK_PARAMS:+ check_param("%s.__init__" % label, cls.__init__, param, None)+ declared = getattr(cls, "model_fields", None)+ if declared is not None:+ check_true("%s/%s declared field" % (label, param), param in declared)+ check_true("%s has no per-call hooks_concurrent" % label,+ "hooks_concurrent" not in inspect.signature(cls.__init__).parameters)++ # --------------------------------------------------------------- usage block+ # `input_tokens` / `output_tokens` are the fields every client decodes, so they are always+ # present. `options` (#538) is additive and conditional: it appears only for a request whose+ # options lost their distinct token spans, which is what keeps it out of ordinary responses.+ from laya.common import build_sequence, collapsed_options # noqa: E402++ check_param("build_sequence", build_sequence, "return_stats", False)+ check("collapsed_options/nothing collapsed is empty",+ collapsed_options(["q"], [{"options": {"options": 3, "options_distinct": 3,+ "tokens_per_option": None}}]), {})+ check("collapsed_options/a collapsed question is reported",+ collapsed_options(["q"], [{"options": {"options": 58, "options_distinct": 42,+ "tokens_per_option": 4}}]),+ {"q": {"total": 58, "distinct": 42, "tokens_per_option": 4}})+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: