Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(integrations): gate the routing hints at every entry point, local and remote

Aashish3a9cd23e9bb6
diff --git a/tests/test_crewai.py b/tests/test_crewai.py
index 58b3cc36..7d488b0d 100644
--- a/tests/test_crewai.py
+++ b/tests/test_crewai.py
@@ -344,6 +344,8 @@ from laya.router import Router
CONTROLS = (tuple(_controls.PREDICT_CONTROLS) + tuple(_controls.DECISION_CONTROLS)
+ tuple(_controls.HOOK_CONTROLS))
+CONTROLS = tuple(_controls.PREDICT_CONTROLS) + tuple(_controls.HOOK_CONTROLS)
+ROUTER_HINTS = tuple(_controls.ROUTER_ONLY_CONTROLS)
def _params(fn):
@@ -357,6 +359,8 @@ check("controls/decision tuple names decision_kwargs",
set(_params(_controls.decision_kwargs)), set(_controls.DECISION_CONTROLS))
check("controls/hook tuple names hook_kwargs",
set(_params(_controls.hook_kwargs)), set(_controls.HOOK_CONTROLS))
+check("controls/routing tuple names router_kwargs",
+ set(_params(_controls.router_kwargs)), set(_controls.ROUTER_ONLY_CONTROLS))
# Everything this file calls a control is an argument a real runner accepts, or a chain would fail
# with a TypeError deep inside core instead of at the call site.
@@ -366,15 +370,23 @@ for _c in CONTROLS:
check_true("controls/%s accepted by Agent" % _c, _c in _agent_params)
check_true("controls/%s accepted by Router.predict" % _c, _c in _router_params)
+# The two routing hints are the opposite case, and the whole reason `require_router_controls`
+# exists: a `Router` reads them and an `Agent` -- which has no routing step to read them in --
+# does not. Read from core rather than asserted in a comment, because a runner that gains either
+# knob turns the refusal below from correct into a bug, and only this check would say so.
+for _c in ROUTER_HINTS:
+ check_true("routing/%s accepted by Router.predict" % _c, _c in _router_params)
+ check_true("routing/%s rejected by Agent.predict" % _c, _c not in _agent_params)
+
# Both classes and the shared executor must accept all of them. Two classes, two call sites --
# a third of them missing is exactly the bug this section exists to keep shut.
for cls in (LayaCrewRouter, LayaTaskGuard):
- for _c in CONTROLS:
+ for _c in CONTROLS + ROUTER_HINTS:
check_true("controls/%s takes %s" % (cls.__name__, _c), _c in _params(cls.__init__))
check("controls/_execute_decision takes every control",
set(_params(crewai_module._execute_decision)) - {"state", "questions", "agent",
"base_url", "api_key", "model"},
- set(CONTROLS))
+ set(CONTROLS) | set(ROUTER_HINTS))
# The three wrappers end at the same runner call, so they must accept the same controls.
for _mod in (langchain_module, llamaindex_module):
@@ -572,6 +584,133 @@ def _silent(state, questions):
kept = LayaCrewRouter(agent=DisagreeingAgent(_silent), confidence_threshold=0.80).route(
"anything", agents)
check("gate/missing confidence still passes", kept.agent_index, 2)
+# ------------------------------------------------------- 5c. The two Router-only routing hints
+ROUTING_ANSWER = {
+ "answers": {
+ "delegation": {"choice": "agent_0", "confidence": 0.9, "answer_confidence": 0.9},
+ "injection_risk": {"type": "noul", "noul": 0.1, "confidence": 0.9},
+ },
+ "routing": {"model": "english", "repo": None, "reason": "explicit model"},
+}
+
+
+class RoutingRunner:
+ """A Router-shaped runner: it declares the arguments core's `Router.predict` declares.
+
+ Explicit parameters rather than `**kwargs`, because the point of the section is the *name* a
+ control arrives under -- a `**kwargs` sink records anything, including a control that arrived
+ misspelled or in the wrong slot.
+ """
+
+ def __init__(self):
+ self.seen = {}
+
+ def predict(self, state, questions, model=None, task=None, lang=None, lang_guess=None,
+ max_len=None, head_max_len=None):
+ self.seen = {k: v for k, v in locals().items()
+ if v is not None and k not in ("self", "state", "questions")}
+ return dict(ROUTING_ANSWER)
+
+
+class NonRoutingRunner(RoutingRunner):
+ """An Agent-shaped runner: the budgets yes, either routing hint never."""
+
+ def predict(self, state, questions, max_len=None, head_max_len=None):
+ return RoutingRunner.predict(self, state, questions, max_len=max_len,
+ head_max_len=head_max_len)
+
+
+# A mock that silently diverges from core would make the two sections below agree with each other
+# and with nothing real. Each declared name has to be one the entry point it imitates accepts.
+check_true("routing/RoutingRunner imitates accepted arguments only",
+ (_params(RoutingRunner.predict) - {"state", "questions"}) <= _router_params,
+ repr(_params(RoutingRunner.predict)))
+check_true("routing/NonRoutingRunner imitates accepted arguments only",
+ (_params(NonRoutingRunner.predict) - {"state", "questions"}) <= _agent_params,
+ repr(_params(NonRoutingRunner.predict)))
+
+SURFACES = (
+ ("router", lambda runner, **kw: LayaCrewRouter(agent=runner, **kw).route(
+ "Analyze Q3 sales", agents)),
+ ("guard", lambda runner, **kw: LayaTaskGuard(agent=runner, **kw).screen(
+ "Summarize this report")),
+)
+
+for _label, _run in SURFACES:
+ def _seen(_run=_run, _cls=RoutingRunner, **_hints):
+ _r = _cls()
+ _run(_r, **_hints)
+ return _r.seen
+
+ check("routing/%s default sends nothing" % _label, _seen(), {})
+ check("routing/%s forwards both hints" % _label,
+ _seen(task="typed_decisions", lang_guess="de"),
+ {"task": "typed_decisions", "lang_guess": "de"})
+ check("routing/%s forwards one hint alone" % _label, _seen(lang_guess="de"),
+ {"lang_guess": "de"})
+ # A blank hint is core's documented "no usable hint, fall through to detection", so it is a
+ # value the caller chose and must survive; `task=""` is not a fall-through, it is an empty name.
+ check("routing/%s keeps an empty lang_guess" % _label, _seen(lang_guess=""),
+ {"lang_guess": ""})
+ check("routing/%s beside model and budget" % _label,
+ _seen(model="laya-multilingual", max_len=1024, task="typed_decisions"),
+ {"model": "laya-multilingual", "max_len": 1024, "task": "typed_decisions"})
+
+ # And a runner with no routing step must not be handed a routing hint: core would raise a
+ # `TypeError` from inside `predict`, past every hook, on the way to an answer nobody asked for.
+ for _c, _sample in zip(ROUTER_HINTS, ("typed_decisions", "de")):
+ try:
+ _seen(_cls=NonRoutingRunner, **{_c: _sample})
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
+ False, "no error raised")
+ except ValueError as exc:
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c), True)
+ check_true("routing/%s %s names itself" % (_label, _c), _c in str(exc))
+ check_true("routing/%s %s names the call that refused" % (_label, _c),
+ "NonRoutingRunner.predict" in str(exc))
+ check_true("routing/%s %s says what to do" % (_label, _c), "Router" in str(exc))
+ except Exception as exc:
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
+ False, type(exc).__name__)
+
+ # Nothing was forwarded, so nothing reached the runner either.
+ _strict = NonRoutingRunner()
+ try:
+ _run(_strict, task="typed_decisions")
+ except ValueError:
+ pass
+ check("routing/%s refusal happens before the call" % _label, _strict.seen, {})
+
+# Both hints ride in the laya-serve body -- they are `BODY_CONTROLS` there -- and a blank one
+# stays blank rather than dropping out of the request.
+check("routing/remote body carries both hints",
+ {k: v for k, v in remote_body({"task": "typed_decisions", "lang_guess": "de"}).items()
+ if k in ROUTER_HINTS},
+ {"task": "typed_decisions", "lang_guess": "de"})
+check("routing/remote body omits unset hints",
+ [k for k in remote_body({}) if k in ROUTER_HINTS], [])
+check("routing/remote body keeps an empty lang_guess",
+ remote_body({"lang_guess": ""}).get("lang_guess"), "")
+
+# A `lang_guess` may also be a callable taking the state, and only the code string has a wire form.
+# `laya-serve` answers 422 for the callable; naming it here says so before the request goes out.
+for _surface, _go in (("router", lambda **kw: LayaCrewRouter(base_url="http://localhost:8000",
+ **kw).route("x", agents)),
+ ("guard", lambda **kw: LayaTaskGuard(base_url="http://localhost:8000",
+ **kw).screen("x"))):
+ try:
+ _go(lang_guess=lambda state: "de")
+ check_true("routing/%s remote refuses a callable lang_guess" % _surface,
+ False, "no error raised")
+ except ValueError as exc:
+ check_true("routing/%s remote refuses a callable lang_guess" % _surface, True)
+ check_true("routing/%s remote lang_guess names itself" % _surface,
+ "lang_guess" in str(exc))
+ check_true("routing/%s remote lang_guess names the endpoint" % _surface,
+ "laya-serve" in str(exc))
+ except Exception as exc:
+ check_true("routing/%s remote refuses a callable lang_guess" % _surface,
+ False, type(exc).__name__)
# --------------------------------------------------------------- Results Summary
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 9b4b0c6d..c4010d0d 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -961,6 +961,16 @@ check_param("evaluate_shortlist", evaluate_shortlist, "k", 20)
check_param("evaluate_shortlist", evaluate_shortlist, "dataset_path", None)
for param in ("checkpoint_id", "embedder_id"):
check_param("evaluate_shortlist", evaluate_shortlist, param, inspect.Parameter.empty)
+# The framework integrations send the two routing hints (`task`, `lang_guess`) in the same body, so
+# "a chain step and a raw HTTP client get the same answer" has to hold of them too. If serve dropped
+# one from `BODY_CONTROLS`, a wrapper sending it would get a 422 its caller cannot fix from the
+# integrations -- this says so at the shared declaration instead of at a request.
+from laya.integrations._controls import ROUTER_ONLY_CONTROLS as _router_only # noqa: E402
+
+check("serve accepts every routing hint an integration sends",
+ [key for key in _router_only if key not in _http_controls], [])
+check("routing hints are not among serve's refusals",
+ sorted(set(_router_only) & set(_http_refusals)), [])
# docs/hooks/examples.md ## Composition teaches one scope ordering contract; the pre-fix wording
diff --git a/tests/test_langchain.py b/tests/test_langchain.py
index fe9580b6..4d60d800 100644
--- a/tests/test_langchain.py
+++ b/tests/test_langchain.py
@@ -1348,6 +1348,285 @@ finally:
langchain_module._call_remote = _real_call_remote
+# ------------------------------------ 8. The two Router-only routing hints, on every entry point
+from laya.integrations import _controls as controls_module # noqa: E402
+
+ROUTER_HINTS = tuple(controls_module.ROUTER_ONLY_CONTROLS)
+
+# The tuples name the arguments their builders accept, in both directions.
+check("routing/routing tuple names router_kwargs",
+ set(inspect.signature(controls_module.router_kwargs).parameters), set(ROUTER_HINTS))
+
+# A `Router` reads both hints and an `Agent` -- which has no routing step to read them in -- reads
+# neither, so the wrapper's rule (forward at one, refuse by name at the other) is only correct
+# while these two lines hold. Read from core, not from a comment: if `Agent.predict` ever gains
+# `task`, the refusals below stop being right, and this is what says so.
+for _c in ROUTER_HINTS:
+ check_true("routing/%s is a Router.predict parameter" % _c, _c in router_params)
+ check_true("routing/%s is not an Agent.predict parameter" % _c, _c not in agent_params)
+
+ROUTING_ANSWER = {"model": "mock", "answers": {
+ "route": {"type": "choice", "choice": "billing", "confidence": 0.9, "probabilities": {}}}}
+
+
+class RoutingRunner:
+ """A Router-shaped runner: it declares the arguments core's `Router.predict` declares.
+
+ Explicit parameters rather than `**kwargs`, because what this section checks is the *name* a
+ hint arrives under -- a `**kwargs` sink records anything, including one that arrived misspelled
+ or in the wrong slot.
+ """
+
+ def __init__(self):
+ self.seen = {}
+
+ def predict(self, state, questions, model=None, task=None, lang=None, lang_guess=None,
+ max_len=None, head_max_len=None):
+ self.seen = {k: v for k, v in locals().items()
+ if v is not None and k not in ("self", "state", "questions")}
+ return dict(ROUTING_ANSWER)
+
+
+class NonRoutingRunner(RoutingRunner):
+ """An Agent-shaped runner: the budgets yes, either routing hint never."""
+
+ def predict(self, state, questions, max_len=None, head_max_len=None):
+ return RoutingRunner.predict(self, state, questions, max_len=max_len,
+ head_max_len=head_max_len)
+
+
+check_true("routing/RoutingRunner declares only accepted arguments",
+ (set(inspect.signature(RoutingRunner.predict).parameters)
+ - {"self", "state", "questions"}) <= router_params)
+check_true("routing/NonRoutingRunner declares only accepted arguments",
+ (set(inspect.signature(NonRoutingRunner.predict).parameters)
+ - {"self", "state", "questions"}) <= agent_params)
+
+# Every node in the family, built around a runner of choosing.
+def _routing_seen(build, runner_cls, **hints):
+ runner = runner_cls()
+ build(runner, **hints).invoke("some state")
+ return runner.seen
+
+
+for name, build in HOOK_NODES:
+ check("routing/%s default sends nothing" % name, _routing_seen(build, RoutingRunner), {})
+ check("routing/%s forwards both hints" % name,
+ _routing_seen(build, RoutingRunner, task="typed_decisions", lang_guess="de"),
+ {"task": "typed_decisions", "lang_guess": "de"})
+ check("routing/%s forwards one hint alone" % name,
+ _routing_seen(build, RoutingRunner, lang_guess="de"), {"lang_guess": "de"})
+ # A blank hint is core's documented "no usable hint, fall through to detection" -- a value the
+ # caller chose, so it must survive rather than drop out as if unset.
+ check("routing/%s keeps an empty lang_guess" % name,
+ _routing_seen(build, RoutingRunner, lang_guess=""), {"lang_guess": ""})
+ check("routing/%s beside model and budget" % name,
+ _routing_seen(build, RoutingRunner, model="laya-multilingual", max_len=1024,
+ task="typed_decisions"),
+ {"model": "laya-multilingual", "max_len": 1024, "task": "typed_decisions"})
+
+ # A runner with no routing step must not be handed a routing hint. Forwarding it is a
+ # `TypeError` raised inside core, past every hook, on the way to an answer nobody asked for;
+ # dropping it is the silence this whole module exists to break.
+ for _c, _sample in zip(ROUTER_HINTS, ("typed_decisions", "de")):
+ raised, message = False, ""
+ try:
+ _routing_seen(build, NonRoutingRunner, **{_c: _sample})
+ except ValueError as e:
+ raised, message = True, str(e)
+ check_true("routing/%s refuses %s at a runner that cannot route" % (name, _c), raised)
+ check_true("routing/%s %s names itself" % (name, _c), _c in message)
+ check_true("routing/%s %s names the call that refused" % (name, _c),
+ "NonRoutingRunner.predict" in message)
+ check_true("routing/%s %s says what to do instead" % (name, _c), "Router" in message)
+
+ # The refusal happens before the runner is called, so a hint that cannot be honoured cannot
+ # also carry a partially-applied call with it.
+ strict = NonRoutingRunner()
+ try:
+ build(strict, task="typed_decisions").invoke("some state")
+ except ValueError:
+ pass
+ check("routing/%s refusal happens before the call" % name, strict.seen, {})
+
+# Each constructor has to accept both names, or the two checks above are vacuous: `extra` is
+# "allow" on these runnables, so a parameter the class forgot is still accepted at construction,
+# still reads back as a normal attribute, and is still dropped at the call. That is the shape of
+# the bug this section closes, and why declaring the field is checked separately from storing it.
+for cls in (LayaRouter, LayaGuardrail, LayaTriage, LayaEvaluator):
+ for _c in ROUTER_HINTS:
+ check_true("routing/%s __init__ takes %s" % (cls.__name__, _c),
+ _c in inspect.signature(cls.__init__).parameters)
+ check_true("routing/%s declares %s as a field" % (cls.__name__, _c),
+ _c in cls.__annotations__)
+
+# The remote path: both hints are laya-serve body controls, so they ride in the JSON, and a blank
+# one stays in it. `LayaDecision` is not in this family -- it bypasses `_execute_decision` and calls
+# `laya.decide` against a schema, so it has its own forwarding to make.
+remote_routing_calls = []
+_real_remote = langchain_module._call_remote
+
+
+def routing_spy_call_remote(base_url, state, questions, api_key=None, model=None, **extras):
+ remote_routing_calls.append(extras)
+ return {"answers": {"route": {"type": "choice", "choice": "billing", "confidence": 0.9}}}
+
+
+REMOTE_NODES = (
+ ("router", lambda **kw: LayaRouter(HOOK_CRITERIA, base_url="http://laya:8000", **kw)),
+ ("guardrail", lambda **kw: LayaGuardrail(questions=HOOK_QUESTIONS,
+ base_url="http://laya:8000", **kw)),
+ ("triage", lambda **kw: LayaTriage(base_url="http://laya:8000", **kw)),
+ ("evaluator", lambda **kw: LayaEvaluator(
+ questions={"faithful": {"type": "noul", "instructions": "faithful?"}},
+ base_url="http://laya:8000", **kw)),
+)
+
+langchain_module._call_remote = routing_spy_call_remote
+try:
+ for name, build in REMOTE_NODES:
+ remote_routing_calls.clear()
+ build().invoke("some state")
+ check("routing/%s remote sends nothing by default" % name, remote_routing_calls[-1], {})
+
+ remote_routing_calls.clear()
+ build(task="typed_decisions", lang_guess="de").invoke("some state")
+ check("routing/%s remote body carries both hints" % name, remote_routing_calls[-1],
+ {"task": "typed_decisions", "lang_guess": "de"})
+
+ remote_routing_calls.clear()
+ build(lang_guess="").invoke("some state")
+ check("routing/%s remote body keeps an empty lang_guess" % name,
+ remote_routing_calls[-1], {"lang_guess": ""})
+
+ # A `lang_guess` may also be a callable taking the state, and only the code string has a
+ # wire form: `laya-serve` validates it as a string and answers 422 to anything else, so the
+ # callable would fail as an HTTP status after the request had already gone out.
+ raised, message = False, ""
+ try:
+ build(lang_guess=lambda state: "de").invoke("some state")
+ except ValueError as e:
+ raised, message = True, str(e)
+ check_true("routing/%s remote refuses a callable lang_guess" % name, raised)
+ check_true("routing/%s callable lang_guess names itself" % name, "lang_guess" in message)
+ check_true("routing/%s callable lang_guess names the endpoint" % name,
+ "laya-serve" in message)
+ check("routing/remote made no call for a refused hint", len(remote_routing_calls), 1)
+finally:
+ langchain_module._call_remote = _real_remote
+
+# The real `_call_remote` has to put them in the JSON body, not merely accept the keywords.
+_routing_sent = {}
+
+
+class _RoutingFakeOpener:
+ def open(self, req, timeout=None):
+ _routing_sent["body"] = json.loads(req.data.decode("utf-8"))
+ return _FakeResponse()
+
+
+_real_opener = langchain_module.urllib.request.build_opener
+langchain_module.urllib.request.build_opener = lambda *a, **k: _RoutingFakeOpener()
+try:
+ langchain_module._call_remote("http://laya:8000", "x", {}, task="typed_decisions",
+ lang_guess="de")
+ check("routing/_call_remote body carries task", _routing_sent["body"].get("task"),
+ "typed_decisions")
+ check("routing/_call_remote body carries lang_guess",
+ _routing_sent["body"].get("lang_guess"), "de")
+ langchain_module._call_remote("http://laya:8000", "x", {})
+ check_true("routing/_call_remote omits unset hints",
+ "task" not in _routing_sent["body"] and "lang_guess" not in _routing_sent["body"])
+finally:
+ langchain_module.urllib.request.build_opener = _real_opener
+
+
+# The batch entry points, which reach core through a different door: a `Router` gets one request
+# dict per state and `route_batch` hands each dict's hints to `route`, while an `Agent` gets the
+# hints as call arguments it does not take. Both are checked at the signature of that door.
+class RoutingRouterLike(MockRouterLike):
+ """Router-shaped, and it declares `route` the way core does: hints in, no `**kwargs`."""
+
+ def __init__(self, response_fn):
+ super().__init__(response_fn)
+ self.routed = []
+
+ def route(self, state, questions, model=None, task=None, lang=None, lang_guess=None):
+ self.routed.append({k: v for k, v in locals().items()
+ if v is not None and k not in ("self", "state", "questions")})
+ return {"model": "english"}
+
+
+class NonRoutingBatchAgent(MockBatchAgent):
+ """`predict_batch(states, questions, ...)` with core's Agent argument list -- no hints."""
+
+ def predict_batch(self, states, questions, batch_size=None, lang=None, max_len=None,
+ head_max_len=None, sort_by_length=False):
+ return super().predict_batch(states, questions, batch_size=batch_size, lang=lang,
+ max_len=max_len, head_max_len=head_max_len,
+ sort_by_length=sort_by_length)
+
+
+check_true("routing/RoutingRouterLike.route declares only accepted arguments",
+ (set(inspect.signature(RoutingRouterLike.route).parameters)
+ - {"self", "state", "questions"}) <= set(inspect.signature(Router.route).parameters))
+
+_router_shape = RoutingRouterLike(mock_router_response)
+_node = LayaRouter(HOOK_CRITERIA, agent=_router_shape, task="typed_decisions", lang_guess="de")
+check("routing/batch on a Router shapes the answers", _node.batch(ROUTER_INPUTS),
+ [_node.invoke(text) for text in ROUTER_INPUTS])
+check("routing/batch Router carries both hints per request",
+ {k: v for k, v in _router_shape.batch_calls[-1]["requests"][0].items()
+ if k in ROUTER_HINTS},
+ {"task": "typed_decisions", "lang_guess": "de"})
+
+_plain_shape = RoutingRouterLike(mock_router_response)
+LayaRouter(HOOK_CRITERIA, agent=_plain_shape).batch(ROUTER_INPUTS)
+check("routing/batch Router sends nothing by default",
+ [{k: v for k, v in request.items() if k in ROUTER_HINTS}
+ for request in _plain_shape.batch_calls[-1]["requests"]],
+ [{} for _ in ROUTER_INPUTS])
+
+_agent_shape = NonRoutingBatchAgent(mock_router_response)
+try:
+ LayaRouter(HOOK_CRITERIA, agent=_agent_shape, task="typed_decisions").batch(ROUTER_INPUTS)
+ check_true("routing/batch refuses task at a runner that cannot route", False, "no error raised")
+except ValueError as e:
+ check_true("routing/batch refuses task at a runner that cannot route", True)
+ check_true("routing/batch refusal names the batch entry point",
+ "NonRoutingBatchAgent.predict_batch" in str(e), str(e))
+ check("routing/batch refusal made no call", _agent_shape.batch_calls, [])
+
+# A duck-typed batch runner that takes `**kwargs` is not refused: this module cannot tell what it
+# reads, and refusing on a guess would block a call that works.
+_kwargs_batch = MockBatchAgent(mock_router_response)
+LayaRouter(HOOK_CRITERIA, agent=_kwargs_batch, task="typed_decisions").batch(ROUTER_INPUTS)
+check("routing/batch forwards to a **kwargs runner",
+ _kwargs_batch.batch_calls[-1]["kwargs"].get("task"), "typed_decisions")
+
+
+class RouteIgnoringHintsRouter(MockRouterLike):
+ """Router-shaped, but its routing step reads neither hint.
+
+ This is the witness that the Router batch path asks about the door the hints actually go
+ through: `predict_batch(requests)` takes no routing argument at all and would accept anything,
+ so checking that signature instead of `route`'s would forward in silence.
+ """
+
+ def route(self, state, questions, model=None, lang=None):
+ return {"model": "english"}
+
+
+_hints_free_router = RouteIgnoringHintsRouter(mock_router_response)
+try:
+ LayaRouter(HOOK_CRITERIA, agent=_hints_free_router, task="typed_decisions").batch(
+ ROUTER_INPUTS)
+ check_true("routing/batch Router asks about route", False, "no error raised")
+except ValueError as exc:
+ check_true("routing/batch Router asks about route",
+ "RouteIgnoringHintsRouter.route" in str(exc), str(exc))
+
+
# --------------------------------------------------------------- Summary
print(f"PASS: {len(PASS)}")
print(f"FAIL: {len(FAIL)}")
diff --git a/tests/test_llamaindex.py b/tests/test_llamaindex.py
index 232b1587..9df1b6b2 100644
--- a/tests/test_llamaindex.py
+++ b/tests/test_llamaindex.py
@@ -411,6 +411,7 @@ from laya.router import Router
CONTROLS = (tuple(_controls.PREDICT_CONTROLS) + tuple(_controls.DECISION_CONTROLS)
+ tuple(_controls.HOOK_CONTROLS))
+ROUTER_HINTS = tuple(_controls.ROUTER_ONLY_CONTROLS)
def _params(fn):
@@ -423,6 +424,8 @@ check("controls/decision tuple names decision_kwargs",
set(_params(_controls.decision_kwargs)), set(_controls.DECISION_CONTROLS))
check("controls/hook tuple names hook_kwargs",
set(_params(_controls.hook_kwargs)), set(_controls.HOOK_CONTROLS))
+check("controls/routing tuple names router_kwargs",
+ set(_params(_controls.router_kwargs)), set(_controls.ROUTER_ONLY_CONTROLS))
# Everything called a control here is an argument a real runner accepts, or a selector would fail
# with a TypeError deep inside core instead of at the call site.
@@ -432,13 +435,20 @@ for _c in CONTROLS:
check_true("controls/%s accepted by Agent" % _c, _c in _agent_params)
check_true("controls/%s accepted by Router.predict" % _c, _c in _router_params)
+# The two routing hints are the reverse case, and it is read from core rather than written down:
+# a Router has a routing step to read them in and an Agent does not, so the wrapper's rule --
+# forward at one, refuse at the other -- is only correct while these two lines hold.
+for _c in ROUTER_HINTS:
+ check_true("routing/%s accepted by Router.predict" % _c, _c in _router_params)
+ check_true("routing/%s rejected by Agent.predict" % _c, _c not in _agent_params)
+
for cls in (LayaSingleSelector, LayaMultiSelector, LayaQueryRouter):
- for _c in CONTROLS:
+ for _c in CONTROLS + ROUTER_HINTS:
check_true("controls/%s takes %s" % (cls.__name__, _c), _c in _params(cls.__init__))
check("controls/_execute_decision takes every control",
set(_params(llamaindex_module._execute_decision)) - {"state", "questions", "agent",
"base_url", "api_key", "model"},
- set(CONTROLS))
+ set(CONTROLS) | set(ROUTER_HINTS))
# The three wrappers end at the same runner call, so they must accept the same controls.
for _mod in (langchain_module, crewai_module):
@@ -622,6 +632,111 @@ def _silent_selector(state, questions):
kept = LayaSingleSelector(agent=DisagreeingSelectorAgent(_silent_selector)).select(tools, "anything")
check("gate/missing confidence still passes",
kept.selections[0].index if hasattr(kept, "selections") else kept, 2)
+# ------------------------------------------------- 6b. The two Router-only routing hints
+class RoutingRunner:
+ """A Router-shaped runner: it declares the arguments core's `Router.predict` declares.
+
+ Explicit parameters rather than `**kwargs`, because the point of this section is the *name* a
+ control arrives under -- a `**kwargs` sink records anything, including a control that arrived
+ misspelled or in the wrong slot.
+ """
+
+ device = "cpu"
+
+ def __init__(self):
+ self.seen = {}
+
+ def predict(self, state, questions, model=None, task=None, lang=None, lang_guess=None,
+ max_len=None, head_max_len=None):
+ self.seen = {k: v for k, v in locals().items()
+ if v is not None and k not in ("self", "state", "questions")}
+ return dict(ROUTING_ANSWER)
+
+
+class NonRoutingRunner(RoutingRunner):
+ """An Agent-shaped runner: the budgets yes, either routing hint never."""
+
+ def predict(self, state, questions, max_len=None, head_max_len=None):
+ return RoutingRunner.predict(self, state, questions, max_len=max_len,
+ head_max_len=head_max_len)
+
+
+# The canned answers every selector here reads back out of one decision.
+ROUTING_ANSWER = RecordingAgent().predict(None, None)
+
+# A mock that drifts from core would make this section agree with itself and with nothing real, so
+# each declared name has to be one the entry point it imitates accepts.
+check_true("routing/RoutingRunner imitates accepted arguments only",
+ (_params(RoutingRunner.predict) - {"state", "questions"}) <= _router_params,
+ repr(_params(RoutingRunner.predict)))
+check_true("routing/NonRoutingRunner imitates accepted arguments only",
+ (_params(NonRoutingRunner.predict) - {"state", "questions"}) <= _agent_params,
+ repr(_params(NonRoutingRunner.predict)))
+
+
+def routing_kwargs(cls, extra, run, runner_cls, **hints):
+ """One decision through `cls`, and whatever the runner says it was handed."""
+ runner = runner_cls()
+ run(cls(agent=runner, **dict(extra, **hints)))
+ return runner.seen
+
+
+for _label, _cls, _extra, _run in SURFACES:
+ check("routing/%s default sends nothing" % _label,
+ routing_kwargs(_cls, _extra, _run, RoutingRunner), {})
+ check("routing/%s forwards both hints" % _label,
+ routing_kwargs(_cls, _extra, _run, RoutingRunner, task="typed_decisions",
+ lang_guess="de"),
+ {"task": "typed_decisions", "lang_guess": "de"})
+ check("routing/%s forwards one hint alone" % _label,
+ routing_kwargs(_cls, _extra, _run, RoutingRunner, lang_guess="de"), {"lang_guess": "de"})
+ # A blank hint is core's documented "no usable hint, fall through to detection", so it is a
+ # value the caller chose and must survive the trip.
+ check("routing/%s keeps an empty lang_guess" % _label,
+ routing_kwargs(_cls, _extra, _run, RoutingRunner, lang_guess=""), {"lang_guess": ""})
+ check("routing/%s beside model and budget" % _label,
+ routing_kwargs(_cls, _extra, _run, RoutingRunner, model="laya-multilingual",
+ max_len=1024, task="typed_decisions"),
+ {"model": "laya-multilingual", "max_len": 1024, "task": "typed_decisions"})
+
+ # A runner with no routing step must not be handed a routing hint: core would raise a
+ # `TypeError` from inside `predict`, past every hook, on the way to an answer nobody asked for.
+ for _c, _sample in zip(ROUTER_HINTS, ("typed_decisions", "de")):
+ try:
+ routing_kwargs(_cls, _extra, _run, NonRoutingRunner, **{_c: _sample})
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
+ False, "no error raised")
+ except ValueError as exc:
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c), True)
+ check_true("routing/%s %s names itself" % (_label, _c), _c in str(exc))
+ check_true("routing/%s %s names the call that refused" % (_label, _c),
+ "NonRoutingRunner.predict" in str(exc))
+ check_true("routing/%s %s says what to do" % (_label, _c), "Router" in str(exc))
+ except Exception as exc:
+ check_true("routing/%s refuses %s at a runner that cannot route" % (_label, _c),
+ False, type(exc).__name__)
+
+ # Both hints ride in the laya-serve body -- `BODY_CONTROLS` there -- and a blank one stays.
+ check("routing/%s remote body carries both hints" % _label,
+ {k: v for k, v in remote_body(_cls, _extra, _run, task="typed_decisions",
+ lang_guess="de").items() if k in ROUTER_HINTS},
+ {"task": "typed_decisions", "lang_guess": "de"})
+ check("routing/%s remote body omits unset hints" % _label,
+ [k for k in remote_body(_cls, _extra, _run) if k in ROUTER_HINTS], [])
+ check("routing/%s remote body keeps an empty lang_guess" % _label,
+ remote_body(_cls, _extra, _run, lang_guess="").get("lang_guess"), "")
+ # A `lang_guess` may be a callable taking the state, and only the code string has a wire form;
+ # serve answers 422 for a callable, so naming it here beats an HTTP status after the request.
+ try:
+ remote_body(_cls, _extra, _run, lang_guess=lambda state: "de")
+ check_true("routing/%s remote refuses a callable lang_guess" % _label, False,
+ "no error raised")
+ except ValueError as exc:
+ check_true("routing/%s remote refuses a callable lang_guess" % _label,
+ "lang_guess" in str(exc) and "laya-serve" in str(exc))
+ except Exception as exc:
+ check_true("routing/%s remote refuses a callable lang_guess" % _label, False,
+ type(exc).__name__)
# --------------------------------------------------------------- Results Summary