Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #569 from AlKor13/feat/report-option-collapse

Nandakishor1921330a5d08
diff --cc laya/common.py
index 15a4cbe2,08b67db3..404d994c
--- a/laya/common.py
+++ b/laya/common.py
@@@ -144,61 -150,38 +151,90 @@@ def build_sequence
# not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of it
st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]
ids = ids + st + [tok.sep_token_id]
- return ids[:max_len], [m for m in markers if m < max_len]
+ ids, markers = ids[:max_len], [m for m in markers if m < max_len]
+ if not return_stats:
+ return ids, markers
+ # Two options that share a prefix can come out of the cut as the same token span: the marker
+ # count still matches the option count, so the guard in `Agent._encode_state` passes and
+ # nothing downstream can tell that the question lost the ability to name them apart. Counted
+ # on the capped option ids, before assembly: re-slicing the finished sequence cannot close
+ # the last option's span -- it runs on into the serialized state, which differs per request,
+ # so the last option always looks distinguishable however it collided (#538).
+ return ids, markers, {
+ "options": len(opt_ids),
+ "options_distinct": len({tuple(o) for o in opt_ids}),
+ "tokens_per_option": per_option,
+ }
+
+
+ def collapsed_options(qids, items) -> Dict[str, Dict[str, Optional[int]]]:
+ """The questions whose options no longer have a token span each, from per-item stats.
+
+ `total` is the number of options the question defines, not the number of markers that
+ reached the sequence: a report counted from the markers would say "43/58" about a request
+ where 28 options never made it into the input at all.
+ """
+ out = {}
+ for qid, item in zip(qids, items):
+ stats = item.get("options")
+ if stats and stats["options_distinct"] < stats["options"]:
+ out[qid] = {"total": stats["options"], "distinct": stats["options_distinct"],
+ "tokens_per_option": stats["tokens_per_option"]}
+ return out
+class _DynamicMultiheadAttention(nn.MultiheadAttention):
+ """`nn.MultiheadAttention` that keeps the shapes it traces.
+
+ The stock module reshapes the packed projection with sizes captured while tracing, so under
+ the legacy ONNX exporter the traced sequence length becomes a constant and a model exported
+ from a short dummy input only runs at that length. The parameters and the maths are the same
+ here; the reshape uses only constant shape arguments (`chunk` / `unflatten` / `flatten`) and
+ `scaled_dot_product_attention`, the kernel the stock path already uses when weights are not
+ requested.
+ """
+
+ def forward(self, query, key, value, key_padding_mask=None, need_weights=True,
+ attn_mask=None, average_attn_weights=True, is_causal=False):
+ if (need_weights or self.in_proj_weight is None or self.bias_k is not None
+ or self.bias_v is not None
+ or (attn_mask is not None and attn_mask.dtype != torch.bool)):
+ # Weight averaging, additive masks and the optional k/v bias are not on the traced
+ # path; the stock implementation keeps them correct.
+ return super().forward(query, key, value, key_padding_mask=key_padding_mask,
+ need_weights=need_weights, attn_mask=attn_mask,
+ average_attn_weights=average_attn_weights, is_causal=is_causal)
+ if self.batch_first:
+ query, key, value = query.transpose(0, 1), key.transpose(0, 1), value.transpose(0, 1)
+ # (T, B, E) from here, matching the stock module's internals; attention runs on (B, H, T, D).
+ if query is key is value:
+ q, k, v = (part.unflatten(-1, (self.num_heads, self.head_dim)).permute(1, 2, 0, 3)
+ for part in F.linear(query, self.in_proj_weight, self.in_proj_bias).chunk(3, dim=-1))
+ else:
+ embed_dim = query.shape[-1]
+ wq, wk, wv = self.in_proj_weight.split(embed_dim, dim=0)
+ bq, bk, bv = ((None, None, None) if self.in_proj_bias is None
+ else self.in_proj_bias.split(embed_dim, dim=0))
+ q, k, v = (
+ F.linear(t, w, b).unflatten(-1, (self.num_heads, self.head_dim)).permute(1, 2, 0, 3)
+ for t, w, b in ((query, wq, bq), (key, wk, bk), (value, wv, bv))
+ )
+ mask = None
+ if attn_mask is not None:
+ mask = ~attn_mask
+ if key_padding_mask is not None:
+ # `== 0` keeps this correct for a bool mask and for the 0 / -inf float mask the encoder
+ # layer hands over (`F._canonical_mask`), where `~` would not be defined.
+ keep = key_padding_mask[:, None, None, :] == 0
+ mask = keep if mask is None else mask & keep
+ attn = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=is_causal and mask is None,
+ dropout_p=self.dropout if self.training else 0.0)
+ attn = attn.permute(2, 0, 1, 3).flatten(-2)
+ if self.batch_first:
+ attn = attn.transpose(0, 1)
+ return self.out_proj(attn), None
+
+
class DecisionModel(nn.Module):
"""Bidirectional transformer encoder backbone + typed decision head."""
diff --cc laya/onnx_agent.py
index 327dd07a,88ef3355..79939e51
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@@ -478,36 -298,41 +479,36 @@@ class ONNXAgent(HookRegistry)
add_special_tokens=False,
)["input_ids"]
+ items = []
for qid in ids:
- q = self._to_internal(questions[qid])
+ q = internal[qid]
- seq, markers = build_sequence(
+ seq, markers, stats = build_sequence(
self.tok, state, q, max_len, head_max_len,
- truncate_left=truncate_left, state_ids=state_ids,
+ truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
)
- if len(markers) != len(render_options(q)):
- raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
+ n_opts = len(render_options(q))
+ if len(markers) != n_opts:
+ # The markers are placed at absolute positions and `build_sequence` then drops the
+ # ones past `max_len`, so this is about the question fitting in the sequence --
+ # `head_max_len` is how much of it the options were given, and `max_len` is the
+ # ceiling that dropped them. Naming only `head_max_len` pointed at the wrong knob
+ # in both directions: lowering it shortens the option block and can make the
+ # call succeed, while raising it makes the overflow worse.
+ raise ValueError(
+ "question %r: its %d options and question text need %d tokens, more than "
+ "max_len=%d allows once head_max_len=%d is spent on them; lower head_max_len, "
+ "raise max_len, or use fewer options"
+ % (qid, n_opts, len(seq), max_len, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})
+ return items
- b = collate_items([items], self.tok.pad_token_id)
-
- # Prepare ONNX inputs as numpy arrays
- ort_inputs = {
- "input_ids": b["input_ids"].numpy().astype(np.int64),
- "attention_mask": b["attention_mask"].numpy().astype(np.int64),
- "marker_pos": b["marker_pos"].numpy().astype(np.int64),
- "marker_mask": b["marker_mask"].numpy().astype(bool),
- "qtype": b["qtype"].numpy().astype(np.int64),
- }
-
- # Run ONNX inference
- ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)
- logits = ort_outs[0]
- act_logits = ort_outs[1]
-
- # Compute softmax for actions manually in numpy
- act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))
- act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)
-
+ def _decode_answers(self, logits, act, items: List[Dict[str, Any]], ids: List[str],
+ internal: Dict[str, Any], offset: int,
+ lang: Optional[str] = None) -> Dict[str, Any]:
+ """Decode the `len(ids)` head rows starting at `offset` into this state's answers."""
answers = {}
- n_tokens = int(b["attention_mask"].sum())
-
for r, qid in enumerate(ids):
- q = self._to_internal(questions[qid])
+ q = internal[qid]
k = len(items[r]["markers"])
qt = QTYPES[q["t"]]
t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
@@@ -554,81 -379,16 +555,86 @@@
"answer_confidence": ans_conf,
"action": ext,
}
+ return answers
+
+ def _infer_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
+ max_len: Optional[int] = None, head_max_len: Optional[int] = None,
+ lang: Optional[str] = None,
+ batch_size: Optional[int] = None,
+ sort_by_length: bool = False) -> List[Dict[str, Any]]:
+ """Validate once, then encode, collate, run and decode in chunks of `batch_size` states.
+
+ With `sort_by_length`, chunks are formed from length-sorted states inside a lookahead
+ window of eight batches (the same contract as `Agent.predict_batch`), so a batch pads to
+ a shorter maximum; results are returned in input order either way.
+ """
+ from .agent import Agent as _Agent
+
+ ids = list(questions.keys())
+ if not ids:
+ return [{"model": "laya-rl-agent-onnx", "answers": {},
+ "usage": {"input_tokens": 0, "output_tokens": 0}} for _ in states]
+ for qid in ids:
+ _Agent._check_question(qid, questions[qid])
+ internal = {qid: self._to_internal(questions[qid]) for qid in ids}
+ max_len = self.cfg.get("max_len", 512) if max_len is None else max_len
+ head_max_len = self.cfg.get("head_max_len", 192) if head_max_len is None else head_max_len
+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)
+
+ results: List[Dict[str, Any]] = []
+ reorder = sort_by_length and 1 < chunk < len(states)
+ # Bound the tokenized lookahead independently of the input size (same as the torch Agent):
+ # sort within windows of eight batches, using each state's longest encoded question row.
+ window = chunk * 8 if reorder else chunk
+ for start in range(0, len(states), window):
+ part = states[start:start + window]
+ encoded = [self._encode_state(st, ids, internal, max_len, head_max_len) for st in part]
+ order = list(range(len(encoded)))
+ if reorder:
+ order.sort(key=lambda i: max(len(item["ids"]) for item in encoded[i]))
+ window_results: List[Optional[Dict[str, Any]]] = [None] * len(encoded)
+ for offset in range(0, len(order), chunk):
+ indices = order[offset:offset + chunk]
+ per_state_items = [encoded[i] for i in indices]
+
+ b = collate_items(per_state_items, self.tok.pad_token_id)
+
+ # Prepare ONNX inputs as numpy arrays
+ ort_inputs = {
+ "input_ids": b["input_ids"].numpy().astype(np.int64),
+ "attention_mask": b["attention_mask"].numpy().astype(np.int64),
+ "marker_pos": b["marker_pos"].numpy().astype(np.int64),
+ "marker_mask": b["marker_mask"].numpy().astype(bool),
+ "qtype": b["qtype"].numpy().astype(np.int64),
+ }
- usage = {"input_tokens": n_tokens, "output_tokens": 0}
- collapsed = collapsed_options(ids, items)
- if collapsed:
- usage["options"] = collapsed
- return {
- "model": "laya-rl-agent-onnx",
- "answers": answers,
- "usage": usage,
- }
+ # Run ONNX inference
+ ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)
+ logits = ort_outs[0]
+ act_logits = ort_outs[1]
+
+ # Compute softmax for actions manually in numpy
+ act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))
+ act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)
+ att = b["attention_mask"]
+
+ row = 0
+ for index, items in zip(indices, per_state_items):
+ nrows = len(items)
+ n_tokens = int(att[row:row + nrows].sum())
++ usage = {"input_tokens": n_tokens, "output_tokens": 0}
++ # Only when a question lost options to the head budget, as on the torch Agent.
++ collapsed = collapsed_options(ids, items)
++ if collapsed:
++ usage["options"] = collapsed
+ window_results[index] = {
+ "model": "laya-rl-agent-onnx",
+ "answers": self._decode_answers(logits, act, items, ids, internal, row, lang=lang),
- "usage": {"input_tokens": n_tokens, "output_tokens": 0},
++ "usage": usage,
+ }
+ row += nrows
+ results.extend(window_results)
+ return results
def decide(self, state: Union[str, dict, list], schema: Any = None, *,
questions: Optional[Dict[str, Dict[str, Any]]] = None, return_details: bool = False,
diff --cc tests/test_hooks_api.py
index 59b7a193,9d8f0b67..4e8a38ab
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@@ -163,96 -157,21 +163,111 @@@ for label, cls in (("Agent", Agent), ("
for method in ("add_hook", "remove_hook", "hooks_installed"):
check_true("%s/%s exists" % (label, method), callable(getattr(cls, method, None)))
+# The LangChain runnables batch: laya.integrations.langchain's own suite checks what
+# batch() returns, so these lines pin only the caller-visible shape. A rename, or losing
+# the keyword-only return_exceptions, would break LCEL and LangGraph map-reduce silently.
+from laya.integrations.langchain import ( # noqa: E402
+ LayaEvaluator,
+ LayaGuardrail,
+ LayaRouter,
+ LayaTriage,
+)
+
+for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),
+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):
+ for method in ("invoke", "batch", "abatch"):
+ check_true("langchain/%s.%s exists" % (label, method),
+ callable(getattr(cls, method, None)))
+ check("langchain/%s/batch defaults" % label,
+ [(p.name, p.kind.name, p.default) for p in sig(cls.batch).values()],
+ [("self", "POSITIONAL_OR_KEYWORD", inspect.Parameter.empty),
+ ("inputs", "POSITIONAL_OR_KEYWORD", inspect.Parameter.empty),
+ ("config", "POSITIONAL_OR_KEYWORD", None),
+ ("return_exceptions", "KEYWORD_ONLY", False),
+ ("kwargs", "VAR_KEYWORD", inspect.Parameter.empty)])
+ check("langchain/%s/abatch defaults" % label,
+ [(p.name, p.kind.name, p.default) for p in sig(cls.abatch).values()],
+ [(p.name, p.kind.name, p.default) for p in sig(cls.batch).values()])
+
+# --------------------------------------------------------------- LangChain decision node
+# `LayaDecision` is the LCEL form of `laya.decide`, so its constructor has to keep accepting the
+# same schema argument and runner overrides. The module imports without langchain-core installed
+# (the runnable base falls back to plain object), so these checks hold in every CI lane.
+from laya.integrations.langchain import LayaDecision # noqa: E402
+
+for param, default in (("return_details", False), ("state_key", None), ("agent", None),
+ ("base_url", None), ("api_key", None), ("model", None)):
+ check_param("LayaDecision.__init__", LayaDecision.__init__, param, default)
+check_param("LayaDecision.__init__", LayaDecision.__init__, "decision_schema",
+ inspect.Parameter.empty, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+check_true("LayaDecision/invoke exists", callable(getattr(LayaDecision, "invoke", None)))
+check_true("LayaDecision/__all__", "LayaDecision" in laya.__all__)
+check_true("LayaDecision/laya attribute", hasattr(laya, "LayaDecision"))
+check_true("LayaDecision/integrations export",
+ "LayaDecision" in __import__("laya.integrations", fromlist=["__all__"]).__all__)
+
+
+# --------------------------------------------------------------- LangChain token budget
+# `max_len`/`head_max_len` are the per-request knobs that decide how many tokens each option of a
+# choice question gets. Every runnable has to carry both, under the names the core API uses, or a
+# chain has no way to widen a question that overflows the checkpoint's budget.
+from laya.integrations.langchain import ( # noqa: E402
+ LayaEvaluator,
+ LayaGuardrail,
+ LayaRouter,
+ LayaTriage,
+)
+
+for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),
+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):
+ # Without langchain-core the runnables are plain objects, so there is no model schema to
+ # carry the field; only the constructor contract applies in that lane.
+ declared = getattr(cls, "model_fields", None)
+ for param in ("max_len", "head_max_len"):
+ check_param("%s.__init__" % label, cls.__init__, param, None)
+ if declared is not None:
+ check_true("%s/%s declared field" % (label, param), param in declared)
+
+
+# The LangChain runnables have to carry the per-call hook arguments too, or a chain cannot
+# install the cache/guard patterns from docs/hooks/patterns.md on a single node. They default to
+# None (meaning "inherit the runner"), which is what keeps an unset node byte-identical to today.
+# `hooks_concurrent` is deliberately absent: like the core predict surfaces, it is not per-call.
+from laya.integrations.langchain import ( # noqa: E402
+ LayaEvaluator,
+ LayaGuardrail,
+ LayaRouter,
+ LayaTriage,
+)
+
+PER_CALL_HOOK_PARAMS = ("hooks", "on_predict_start", "on_predict_end", "hooks_raise",
+ "hooks_timeout")
+
+for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),
+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):
+ for param in PER_CALL_HOOK_PARAMS:
+ check_param("%s.__init__" % label, cls.__init__, param, None)
+ declared = getattr(cls, "model_fields", None)
+ if declared is not None:
+ check_true("%s/%s declared field" % (label, param), param in declared)
+ check_true("%s has no per-call hooks_concurrent" % label,
+ "hooks_concurrent" not in inspect.signature(cls.__init__).parameters)
+
+ # --------------------------------------------------------------- usage block
+ # `input_tokens` / `output_tokens` are the fields every client decodes, so they are always
+ # present. `options` (#538) is additive and conditional: it appears only for a request whose
+ # options lost their distinct token spans, which is what keeps it out of ordinary responses.
+ from laya.common import build_sequence, collapsed_options # noqa: E402
+
+ check_param("build_sequence", build_sequence, "return_stats", False)
+ check("collapsed_options/nothing collapsed is empty",
+ collapsed_options(["q"], [{"options": {"options": 3, "options_distinct": 3,
+ "tokens_per_option": None}}]), {})
+ check("collapsed_options/a collapsed question is reported",
+ collapsed_options(["q"], [{"options": {"options": 58, "options_distinct": 42,
+ "tokens_per_option": 4}}]),
+ {"q": {"total": 58, "distinct": 42, "tokens_per_option": 4}})
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: