Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(langchain): forward the per-request token budget to every runnable

Aashishaa4d793f0757
diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.py
index f7e5cb9b..f0e7f344 100644
--- a/laya/integrations/langchain.py
+++ b/laya/integrations/langchain.py
@@ -144,6 +144,35 @@ def _get_default_router():
return _DEFAULT_ROUTER
+def _reject_remote_budget(max_len: Optional[int], head_max_len: Optional[int],
+ base_url: Optional[str]) -> None:
+ """Refuse a budget override on a remote node rather than dropping it silently.
+
+ `laya-serve` has no budget field of its own -- it answers within the checkpoint's defaults and
+ rejects an oversized question with 413. The budget decides how many tokens each option gets, so
+ ignoring it can change the label the model returns; a caller must not be told it applied when
+ it could not.
+ """
+ if base_url and (max_len is not None or head_max_len is not None):
+ raise ValueError(
+ "max_len/head_max_len are applied by the local runner and cannot be sent to a "
+ "laya-serve endpoint; raise the budget where serve runs, or drop the override"
+ )
+
+
+def _predict_kwargs(model: Optional[str] = None, max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None) -> Dict[str, Any]:
+ """The per-request overrides a local runner accepts, with the unset ones omitted."""
+ kwargs: Dict[str, Any] = {}
+ if model:
+ kwargs["model"] = model
+ if max_len is not None:
+ kwargs["max_len"] = max_len
+ if head_max_len is not None:
+ kwargs["head_max_len"] = head_max_len
+ return kwargs
+
+
def _execute_decision(
state: Any,
questions: Dict[str, Any],
@@ -151,12 +180,14 @@ def _execute_decision(
base_url: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
+ max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None,
) -> Dict[str, Any]:
if base_url:
+ _reject_remote_budget(max_len, head_max_len, base_url)
return _call_remote(base_url, state, questions, api_key=api_key, model=model)
runner = agent if agent is not None else _get_default_router()
- kwargs = {"model": model} if model else {}
- return runner.predict(state, questions, **kwargs)
+ return runner.predict(state, questions, **_predict_kwargs(model, max_len, head_max_len))
class LayaRouter(RunnableSerializable):
@@ -175,6 +206,8 @@ class LayaRouter(RunnableSerializable):
base_url: Optional[str] = None
api_key: Optional[str] = None
model: Optional[str] = None
+ max_len: Optional[int] = None
+ head_max_len: Optional[int] = None
question_id: str = "route"
last_decision: Optional[Dict[str, Any]] = None
@@ -193,6 +226,8 @@ class LayaRouter(RunnableSerializable):
base_url: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
+ max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None,
**kwargs: Any,
):
if _RUNNABLE_AVAILABLE:
@@ -206,6 +241,8 @@ class LayaRouter(RunnableSerializable):
base_url=base_url,
api_key=api_key,
model=model,
+ max_len=max_len,
+ head_max_len=head_max_len,
**kwargs,
)
else:
@@ -218,6 +255,8 @@ class LayaRouter(RunnableSerializable):
self.base_url = base_url
self.api_key = api_key
self.model = model
+ self.max_len = max_len
+ self.head_max_len = head_max_len
self.question_id = "route"
self.last_decision: Optional[Dict[str, Any]] = None
@@ -238,6 +277,8 @@ class LayaRouter(RunnableSerializable):
base_url=self.base_url,
api_key=self.api_key,
model=self.model,
+ max_len=self.max_len,
+ head_max_len=self.head_max_len,
)
self.last_decision = res
ans = res["answers"][self.question_id]
@@ -271,6 +312,8 @@ class LayaGuardrail(RunnableSerializable):
base_url: Optional[str] = None
api_key: Optional[str] = None
model: Optional[str] = None
+ max_len: Optional[int] = None
+ head_max_len: Optional[int] = None
class Config:
arbitrary_types_allowed = True
@@ -287,6 +330,8 @@ class LayaGuardrail(RunnableSerializable):
base_url: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
+ max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None,
**kwargs: Any,
):
if _RUNNABLE_AVAILABLE:
@@ -300,6 +345,8 @@ class LayaGuardrail(RunnableSerializable):
base_url=base_url,
api_key=api_key,
model=model,
+ max_len=max_len,
+ head_max_len=head_max_len,
**kwargs,
)
else:
@@ -312,6 +359,8 @@ class LayaGuardrail(RunnableSerializable):
self.base_url = base_url
self.api_key = api_key
self.model = model
+ self.max_len = max_len
+ self.head_max_len = head_max_len
def _default_questions(self) -> Dict[str, Any]:
from ..presets import guard_questions
@@ -329,6 +378,8 @@ class LayaGuardrail(RunnableSerializable):
base_url=self.base_url,
api_key=self.api_key,
model=self.model,
+ max_len=self.max_len,
+ head_max_len=self.head_max_len,
)
answers = res.get("answers", {})
@@ -398,6 +449,8 @@ class LayaTriage(RunnableSerializable):
base_url: Optional[str] = None
api_key: Optional[str] = None
model: Optional[str] = None
+ max_len: Optional[int] = None
+ head_max_len: Optional[int] = None
class Config:
arbitrary_types_allowed = True
@@ -410,6 +463,8 @@ class LayaTriage(RunnableSerializable):
base_url: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
+ max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None,
**kwargs: Any,
):
if _RUNNABLE_AVAILABLE:
@@ -419,6 +474,8 @@ class LayaTriage(RunnableSerializable):
base_url=base_url,
api_key=api_key,
model=model,
+ max_len=max_len,
+ head_max_len=head_max_len,
**kwargs,
)
else:
@@ -427,6 +484,8 @@ class LayaTriage(RunnableSerializable):
self.base_url = base_url
self.api_key = api_key
self.model = model
+ self.max_len = max_len
+ self.head_max_len = head_max_len
def invoke(self, state: Any, config: Optional[RunnableConfig] = None) -> Dict[str, Any]:
"""Triage the state and return enriched fields."""
@@ -440,6 +499,8 @@ class LayaTriage(RunnableSerializable):
base_url=self.base_url,
api_key=self.api_key,
model=self.model,
+ max_len=self.max_len,
+ head_max_len=self.head_max_len,
)
ans = res.get("answers", {})
@@ -475,6 +536,8 @@ class LayaEvaluator(RunnableSerializable):
base_url: Optional[str] = None
api_key: Optional[str] = None
model: Optional[str] = None
+ max_len: Optional[int] = None
+ head_max_len: Optional[int] = None
class Config:
arbitrary_types_allowed = True
@@ -488,6 +551,8 @@ class LayaEvaluator(RunnableSerializable):
base_url: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
+ max_len: Optional[int] = None,
+ head_max_len: Optional[int] = None,
**kwargs: Any,
):
if _RUNNABLE_AVAILABLE:
@@ -498,6 +563,8 @@ class LayaEvaluator(RunnableSerializable):
base_url=base_url,
api_key=api_key,
model=model,
+ max_len=max_len,
+ head_max_len=head_max_len,
**kwargs,
)
else:
@@ -507,6 +574,8 @@ class LayaEvaluator(RunnableSerializable):
self.base_url = base_url
self.api_key = api_key
self.model = model
+ self.max_len = max_len
+ self.head_max_len = head_max_len
def evaluate_strings(self, *, prediction: str, input: Optional[str] = None, **kwargs: Any) -> Dict[str, Any]:
"""LangChain standard string evaluation interface."""
@@ -518,6 +587,8 @@ class LayaEvaluator(RunnableSerializable):
base_url=self.base_url,
api_key=self.api_key,
model=self.model,
+ max_len=self.max_len,
+ head_max_len=self.head_max_len,
)
return res.get("answers", {})
@@ -530,6 +601,8 @@ class LayaEvaluator(RunnableSerializable):
base_url=self.base_url,
api_key=self.api_key,
model=self.model,
+ max_len=self.max_len,
+ head_max_len=self.head_max_len,
)
return res.get("answers", {})
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..0a0b4611 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -158,6 +158,28 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgent
check_true("%s/%s exists" % (label, method), callable(getattr(cls, method, None)))
+# --------------------------------------------------------------- LangChain token budget
+# `max_len`/`head_max_len` are the per-request knobs that decide how many tokens each option of a
+# choice question gets. Every runnable has to carry both, under the names the core API uses, or a
+# chain has no way to widen a question that overflows the checkpoint's budget.
+from laya.integrations.langchain import ( # noqa: E402
+ LayaEvaluator,
+ LayaGuardrail,
+ LayaRouter,
+ LayaTriage,
+)
+
+for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),
+ ("LayaTriage", LayaTriage), ("LayaEvaluator", LayaEvaluator)):
+ # Without langchain-core the runnables are plain objects, so there is no model schema to
+ # carry the field; only the constructor contract applies in that lane.
+ declared = getattr(cls, "model_fields", None)
+ for param in ("max_len", "head_max_len"):
+ check_param("%s.__init__" % label, cls.__init__, param, None)
+ if declared is not None:
+ check_true("%s/%s declared field" % (label, param), param in declared)
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)
diff --git a/tests/test_langchain.py b/tests/test_langchain.py
index 628876c0..e86aef7c 100644
--- a/tests/test_langchain.py
+++ b/tests/test_langchain.py
@@ -8,6 +8,7 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from laya.integrations import langchain as langchain_module
from laya.integrations.langchain import (
LayaEvaluator,
LayaGuardrail,
@@ -272,6 +273,97 @@ check("evaluator/faithfulness", eval_res["faithfulness"]["noul"], 0.98)
check("evaluator/hallucination", eval_res["hallucination"]["noul"], 0.02)
+# --------------------------------------------------------------- 7. Per-request token budget
+class BudgetAgent:
+ """Answers every question by type and records the keyword arguments it was called with."""
+
+ def __init__(self):
+ self.kwargs = []
+
+ def predict(self, state, questions, **kwargs):
+ self.kwargs.append(kwargs)
+ answers = {}
+ for qid, question in questions.items():
+ qtype = question.get("type")
+ if qtype == "choice":
+ answers[qid] = {"type": "choice", "choice": list(question["criteria"])[0],
+ "confidence": 0.9, "probabilities": {}}
+ elif qtype == "score":
+ answers[qid] = {"type": "score", "score": 1.0, "confidence": 0.9,
+ "probabilities": {}}
+ else:
+ answers[qid] = {"type": "noul", "noul": 0.1, "confidence": 0.9}
+ return {"model": "mock", "answers": answers}
+
+
+BUDGET_CRITERIA = {"billing": "invoices", "tech": "bugs"}
+BUDGET_NODES = (
+ ("router", lambda a, **kw: LayaRouter(BUDGET_CRITERIA, agent=a, **kw)),
+ ("guardrail", lambda a, **kw: LayaGuardrail(
+ questions={"jailbreak": {"type": "noul", "instructions": "jailbreak?"}}, agent=a, **kw)),
+ ("triage", lambda a, **kw: LayaTriage(agent=a, **kw)),
+ ("evaluator", lambda a, **kw: LayaEvaluator(
+ questions={"faithful": {"type": "noul", "instructions": "faithful?"}}, agent=a, **kw)),
+)
+
+for name, build in BUDGET_NODES:
+ # Nothing set has to mean nothing sent, so core's own defaults stay in charge.
+ plain = BudgetAgent()
+ build(plain).invoke("some state")
+ check("budget/%s default sends nothing" % name, plain.kwargs[0], {})
+
+ both = BudgetAgent()
+ build(both, max_len=1024, head_max_len=512).invoke("some state")
+ check("budget/%s forwards both" % name, both.kwargs[0], {"max_len": 1024, "head_max_len": 512})
+
+ # Each knob has to work alone; a budget built from one must not carry the other.
+ one = BudgetAgent()
+ build(one, max_len=2048).invoke("some state")
+ check("budget/%s forwards max_len alone" % name, one.kwargs[0], {"max_len": 2048})
+
+ # A zero budget is a real value, not an absent one.
+ zero = BudgetAgent()
+ build(zero, head_max_len=0).invoke("some state")
+ check("budget/%s keeps head_max_len=0" % name, zero.kwargs[0], {"head_max_len": 0})
+
+ # model= keeps its slot next to the budget.
+ mixed = BudgetAgent()
+ build(mixed, model="laya-multilingual", head_max_len=256).invoke("some state")
+ check("budget/%s with model" % name, mixed.kwargs[0],
+ {"model": "laya-multilingual", "head_max_len": 256})
+
+
+# `laya-serve` has no budget field, so a remote node must refuse rather than drop the override.
+budget_remote_calls = []
+_budget_real_call_remote = langchain_module._call_remote
+
+
+def budget_spy_call_remote(base_url, state, questions, api_key=None, model=None):
+ budget_remote_calls.append({"model": model})
+ return {"answers": {"route": {"type": "choice", "choice": "billing", "confidence": 0.9}}}
+
+
+langchain_module._call_remote = budget_spy_call_remote
+try:
+ remote_plain = LayaRouter(criteria={"billing": "invoices", "tech": "bugs"},
+ base_url="http://laya:8000", api_key="k")
+ check("budget/remote without an override", remote_plain.invoke("x"), "billing")
+ check("budget/remote still sends model", budget_remote_calls[-1], {"model": None})
+
+ for field in ("max_len", "head_max_len"):
+ raised, message = False, ""
+ try:
+ LayaRouter(BUDGET_CRITERIA, base_url="http://laya:8000", **{field: 512}).invoke("x")
+ except ValueError as e:
+ raised, message = True, str(e)
+ check_true("budget/remote refuses %s" % field, raised)
+ check_true("budget/remote %s names the endpoint" % field, "laya-serve" in message)
+
+ check("budget/remote made no extra call", len(budget_remote_calls), 1)
+finally:
+ langchain_module._call_remote = _budget_real_call_remote
+
+
# --------------------------------------------------------------- Summary
print(f"PASS: {len(PASS)}")
print(f"FAIL: {len(FAIL)}")