Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cli): judge the --sort-by-length note per checkpoint group, not per file (#1111)
akushonkamena69fa97df5a3
diff --git a/docs/hooks/api.md b/docs/hooks/api.mdindex f8414403..09bf8230 100644--- a/docs/hooks/api.md+++ b/docs/hooks/api.md@@ -300,7 +300,7 @@ router.predict(state, questions, model=None, task=None, lang=None, lang_guess=Norouter.predict_batch(requests, batch_size=None, hooks_timeout=None, min_confidence=None,sort_by_length=False, hooks=None, on_predict_start=None, on_predict_end=None,- hooks_raise=None)+ hooks_raise=None, diagnostics=None)router.route_batch(requests, hooks_timeout=None, *, hooks=None, hooks_raise=None)diff --git a/laya/cli.py b/laya/cli.pyindex 89857b4f..0b15b956 100644--- a/laya/cli.py+++ b/laya/cli.py@@ -26,6 +26,7 @@ instead of changing it (it needs an answering mode: routing has no confidence to"""import argparse+import inspectimport jsonimport sys@@ -306,6 +307,49 @@ def read_batch_lines(source):return [line.strip() for line in lines if line.strip()]+def _takes_predict_diagnostics(router) -> bool:+ """Whether `router`'s predict_batch takes the per-group sort report of #1111.++ The same signature check the evals runner makes for `sort_by_length`, but exact-name only:+ the report is a contract with core's own Router, and a `**kwargs` forwarder would swallow+ the dict while staying as blind as any other foreign router. Routers without the parameter+ keep the pre-run note, whose file-wide wording is exact for the single group they predict.+ """+ fn = getattr(router, "predict_batch", None)+ if fn is None:+ return False+ try:+ parameters = inspect.signature(fn).parameters+ except (TypeError, ValueError):+ return False+ return "diagnostics" in parameters+++def _note_sort_report(report, total):+ """Say what actually reordered, once core has reported it per checkpoint group."""+ stats = report.get("sort_by_length") or {}+ groups = stats.get("groups") or 0+ if not groups or stats.get("reordered", 0) >= groups:+ return+ smallest, largest = stats.get("smallest"), stats.get("largest")+ if groups == 1:+ # One group saw every line, so the pre-run wording is exact there; keep its copy, with+ # the group's own size rather than the file's, in case a cache hit shrank the batch.+ print("note: --sort-by-length only takes effect with --batch-size N where "+ "1 < N < %d; the states still go in one forward pass" % smallest,+ file=sys.stderr)+ return+ span = "%d" % smallest if smallest == largest else "%d-%d" % (smallest, largest)+ # The post-run counterpart of the pre-run note: instead of guessing from the file before+ # the run, name the groups that could not reorder and the size that would have reached them.+ print("note: --sort-by-length groups within each checkpoint batch, not across the file: "+ "%d of %d batches (%s states) could not reorder; a batch reorders only when "+ "--batch-size is smaller than its state count, and the smallest batch holds %d of "+ "the file's %d lines"+ % (groups - stats["reordered"], groups, span, smallest, total),+ file=sys.stderr)++def run_batch(lines, args, router=None):"""Route or predict a whole file in one call, so states share checkpoint loads andforward passes; returns 0 on success, 2 on a handled error."""@@ -322,19 +366,30 @@ def run_batch(lines, args, router=None):requests = [{"state": {key: line}, "questions": questions, **overrides}for line in lines]kwargs = {"batch_size": args.batch_size, **abstention_override(args)}+ sort_report = Noneif args.sort_by_length:# Sent only when asked for, as Router.predict_batch itself forwards it: a stub or# an attached agent that predates the knob keeps working for every run that does# not request it.kwargs["sort_by_length"] = Truechunk = args.batch_size if args.batch_size and args.batch_size > 0 else len(lines)- if not 1 < chunk < len(lines):+ if _takes_predict_diagnostics(router):+ # The reorder switches per checkpoint/question group inside core, so the+ # file's line count is only an upper bound on the denominators that decide+ # it (#1111): a file that routes to several checkpoints can reorder nothing+ # while `1 < chunk < len(lines)` holds. Ask core for the per-group outcome+ # and note after the run, when the groups are known.+ sort_report = {}+ kwargs["diagnostics"] = sort_report+ elif not 1 < chunk < len(lines):# Core documents this as a no-op rather than an error, so the shell says why# the run was not reordered instead of printing a silent same-speed result.print("note: --sort-by-length only takes effect with --batch-size N where ""1 < N < %d; the states still go in one forward pass" % len(lines),file=sys.stderr)results = router.predict_batch(requests, **kwargs)+ if sort_report is not None:+ _note_sort_report(sort_report, len(lines))for line, result in zip(lines, results):if args.json:print(json.dumps(result, ensure_ascii=False, default=str))diff --git a/laya/mcp/remote.py b/laya/mcp/remote.pyindex c5f7efc9..9951739b 100644--- a/laya/mcp/remote.py+++ b/laya/mcp/remote.py@@ -203,6 +203,7 @@ class RemoteRouter(Router):on_predict_start=None,on_predict_end=None,hooks_raise: Optional[bool] = None,+ diagnostics: Optional[Dict[str, Any]] = None,) -> List[Dict[str, Any]]:"""One ``/v1/systemone`` call per request, answers in input order.@@ -210,7 +211,9 @@ class RemoteRouter(Router):while a Router batch lets each request carry its own questions and controls, so the generalform is the per-request call. ``batch_size`` and ``sort_by_length`` shape forward passes andhave no meaning here; they are accepted and ignored so callers written against ``Router``- keep working. The server applies its own batching inside each call.+ keep working, and ``diagnostics`` is the same kind of tolerance: a remote run has no local+ checkpoint grouping to report (#1111), so the dict is accepted and left untouched.+ The server applies its own batching inside each call."""results: List[Dict[str, Any]] = []if hooks_timeout is not None:diff --git a/laya/router.py b/laya/router.pyindex adc69d77..430ef9a6 100644--- a/laya/router.py+++ b/laya/router.py@@ -1599,6 +1599,7 @@ class Router(HookRegistry):on_predict_start=None,on_predict_end=None,hooks_raise: Optional[bool] = None,+ diagnostics: Optional[Dict[str, Any]] = None,) -> List[Dict[str, Any]]:"""Route and execute a heterogeneous request batch with minimal model churn.@@ -1641,6 +1642,13 @@ class Router(HookRegistry):on_predict_start (PredictHookArg): Plain callable or sequence of callables for start events.on_predict_end (PredictHookArg): Plain callable or sequence of callables for end events.hooks_raise: Override the Router's ``hooks_raise`` policy for this call.+ diagnostics: Optional dict that receives the per-group sort outcome under the key+ ``"sort_by_length"`` once the batch has run: how many question groups actually+ reached an agent, how many of those could reorder, and the smallest / largest+ group size. The reorder switches per group inside this call, so a caller that+ wants to say whether the knob did anything (#1111) reads this instead of+ guessing from the request count beforehand. Untouched when ``sort_by_length``+ is false or no group reached an agent.Returns:One normal Router prediction result per request, in the same order as the input.@@ -1664,6 +1672,11 @@ class Router(HookRegistry):# `predict` returns as it is.results: List[Any] = [None] * len(requests)answered = 0+ # The per-group sort outcome, kept only for a caller that asked for it (#1111): the+ # reorder switches per question group inside the loop below, and no caller outside+ # this method can see the group sizes it forms.+ sort_report = ({"groups": 0, "reordered": 0, "smallest": None, "largest": None}+ if sort_by_length and diagnostics is not None else None)# `compose_hooks`, not `list(self.hooks)`: the composition `predict` uses. It merges# `set_default_hooks` defaults, then installed hooks, then this call's `hooks`,# `on_predict_start` and `on_predict_end`. Reading the instance list alone silently@@ -1783,6 +1796,22 @@ class Router(HookRegistry):% (len(batch_results), len(items)))+ if sort_report is not None:+ # What this group actually did with the knob: the agent reorders only+ # when `batch_size` clears its own `1 < chunk < len(states)` gate+ # (agent.py:1519, onnx_agent.py:761) over this group's states. A group+ # whose agent predates the knob (#294) reports as unsorted, not as an+ # error -- the run answered, just at one state's width.+ n = len(items)+ reorders = bool("sort_by_length" in batch_kwargs and batch_size+ and batch_size > 1 and batch_size < n)+ sort_report["groups"] += 1+ sort_report["reordered"] += int(reorders)+ if sort_report["smallest"] is None or n < sort_report["smallest"]:+ sort_report["smallest"] = n+ if sort_report["largest"] is None or n > sort_report["largest"]:+ sort_report["largest"] = n+for (i, ctx), result in zip(items, batch_results):result["routing"] = dict(decisions[i])ctx.results = [result]@@ -1821,6 +1850,8 @@ class Router(HookRegistry):if answered != len(requests):raise RuntimeError("internal error: batch execution did not produce every result")+ if sort_report is not None:+ diagnostics["sort_by_length"] = sort_reportreturn resultspredict_many = predict_batchdiff --git a/tests/test_cli.py b/tests/test_cli.pyindex 0abd6059..1994c0cf 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -408,6 +408,68 @@ check("sort: batch routing is untouched by a forward-pass knob",code == 0 and stub.predict_batch_calls == [] and len(stub.route_batch_calls) == 1,"predict=%r route=%r" % (stub.predict_batch_calls, len(stub.route_batch_calls)))+# #1111: the two denominators split when routing groups. Six lines that route to two checkpoints+# reach `Agent.predict_batch` as groups of 4 and 2 with the same `--batch-size` forwarded to each,+# so the reorder switches per group while the note used to read the file -- a batch wider than a+# group reordered nothing, silently. This router splits the way core does and reports the per-group+# outcome the way core now does, so the post-run note is checkable without checkpoints.+class GroupingRouter:+ """One agent-sized call per checkpoint group, plus core's sort report."""++ def __init__(self, sizes):+ self.sizes = sizes+ self.calls = []++ def route_batch(self, requests):+ return [StubDecision() for _ in requests]++ def predict_batch(self, requests, batch_size=None, sort_by_length=False, diagnostics=None):+ report = {"groups": 0, "reordered": 0, "smallest": None, "largest": None}+ start = 0+ for size in self.sizes:+ n = len(requests[start:start + size])+ self.calls.append((n, batch_size, sort_by_length))+ reorders = bool(sort_by_length and batch_size and batch_size > 1 and batch_size < n)+ report["groups"] += 1+ report["reordered"] += int(reorders)+ report["smallest"] = n if report["smallest"] is None else min(report["smallest"], n)+ report["largest"] = n if report["largest"] is None else max(report["largest"], n)+ start += size+ if diagnostics is not None:+ diagnostics["sort_by_length"] = report+ return [{"model": "laya-rl", "answers": {"difficulty": {"score": float(i)}}, "usage": {}}+ for i in range(len(requests))]+++six = os.path.join(tmp, "six.txt")+with open(six, "w", encoding="utf-8") as handle:+ handle.write("one\ntwo\nthree\nfour\nfive\nsix\n")++code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "5",+ "--sort-by-length"], router=GroupingRouter([4, 2]))+check("sort: a split batch that reorders nothing is not silent any more",+ code == 0 and stub.calls == [(4, 5, True), (2, 5, True)]+ and "2 of 2 batches (2-4 states)" in err,+ "code %r calls %r err %r" % (code, stub.calls, err))+check("sort: the per-group note does not quote the file as the denominator",+ "1 < N < 6" not in err, err)++code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "3",+ "--sort-by-length"], router=GroupingRouter([4, 2]))+check("sort: a batch between the group sizes says which half missed out",+ code == 0 and "1 of 2 batches (2-4 states)" in err, "err %r" % err)++code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "2",+ "--sort-by-length"], router=GroupingRouter([3, 3]))+check("sort: a batch below every group reorders everywhere and stays quiet",+ code == 0 and err == "" and stub.calls == [(3, 2, True), (3, 2, True)],+ "code %r calls %r err %r" % (code, stub.calls, err))++code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "6",+ "--sort-by-length"], router=GroupingRouter([6]))+check("sort: one group keeps the file-wide wording, sized by the group core saw",+ code == 0 and "1 < N < 6" in err, "err %r" % err)+# A flag nobody can find is a flag nobody uses. README is where `laya --batch` and `/predict/batch`# are taught, so the same pages that teach `--batch-size` have to teach the grouping that makes a# bounded pass cheap -- including the condition it needs, since a run without it changes nothing.diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex 9b4b0c6d..bebdb456 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -158,6 +158,8 @@ check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)check_param("Agent.predict_batch", Agent.predict_batch, "sort_by_length", False)check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)check_param("Router.predict_batch", Router.predict_batch, "sort_by_length", False)+# the #1111 report: an out-dict the batch fills with the per-group sort outcome+check_param("Router.predict_batch", Router.predict_batch, "diagnostics", None)# per-call token-budget overridesfor label, fn in (("Agent.predict_batch", Agent.predict_batch),diff --git a/tests/test_router.py b/tests/test_router.pyindex 6e4efc84..80ee9ef1 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -2065,6 +2065,69 @@ check("near-typo/warnings-as-errors leaves the router unchanged",(dict(_roll.models), dict(_roll.descriptions), dict(_roll.sha256_digests)), _roll_before)+# --------------------------------------------------------------------- sort report (#1111)+# The reorder switches per checkpoint/question group inside predict_batch, on each group's own+# state count -- a denominator no caller outside can see, which is why the shell's old note read+# the file and went quiet exactly when auto-routing split a batch into smaller groups. The+# `diagnostics` out-dict hands the per-group outcome back once the batch has run. Explicit+# `model` overrides split the batch without loading anything and `load` is stubbed, so this+# runs with no checkpoints.+class _SortAgent:+ def __init__(self, name):+ self.name = name+ self.calls = []++ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False):+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)+ self.calls.append((len(states), batch_size, sort_by_length,+ bool(sort_by_length and 1 < chunk < len(states))))+ return [{"model": self.name, "answers": {}, "usage": {}} for _ in states]+++_sr = Router()+_sr_agents = {}+++def _sr_load(name, _rr=_sr):+ key = _rr.resolve(name)+ if key not in _sr_agents:+ _sr_agents[key] = _SortAgent(key)+ return _sr_agents[key]+++_sr.load = _sr_load+# Six requests like the issue's file: four that route to one checkpoint, two to another.+_sr_requests = [{"state": {"body": "line %d" % i}, "questions": Q_GENERIC,+ "model": "english" if i < 4 else "multilingual"} for i in range(6)]++_report = {}+_sr.predict_batch(_sr_requests, batch_size=5, sort_by_length=True, diagnostics=_report)+check("sort report/two groups, a batch wider than both reorders neither", _report,+ {"sort_by_length": {"groups": 2, "reordered": 0, "smallest": 2, "largest": 4}})+check("sort report/each group still got the knob and kept its own size",+ [_sr_agents["english"].calls, _sr_agents["multilingual"].calls],+ [[(4, 5, True, False)], [(2, 5, True, False)]])++for _a in _sr_agents.values():+ _a.calls = []+_report = {}+_sr.predict_batch(_sr_requests, batch_size=3, sort_by_length=True, diagnostics=_report)+check("sort report/a batch between the group sizes reorders exactly one", _report,+ {"sort_by_length": {"groups": 2, "reordered": 1, "smallest": 2, "largest": 4}})+check("sort report/only the group that cleared the gate reordered",+ [_sr_agents["english"].calls, _sr_agents["multilingual"].calls],+ [[(4, 3, True, True)], [(2, 3, True, False)]])++_report = {}+_sr.predict_batch(_sr_requests, batch_size=2, diagnostics=_report)+check("sort report/untouched when the knob is off", _report, {})++check("sort report/results keep input order regardless",+ [r["routing"]["model"] for r in _sr.predict_batch(_sr_requests, batch_size=3,+ sort_by_length=True)],+ ["english"] * 4 + ["multilingual"] * 2)++# --------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: