Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cli): judge the --sort-by-length note per checkpoint group, not per file (#1111)

akushonkamena69fa97df5a3
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index f8414403..09bf8230 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -300,7 +300,7 @@ router.predict(state, questions, model=None, task=None, lang=None, lang_guess=No
router.predict_batch(requests, batch_size=None, hooks_timeout=None, min_confidence=None,
sort_by_length=False, hooks=None, on_predict_start=None, on_predict_end=None,
- hooks_raise=None)
+ hooks_raise=None, diagnostics=None)
router.route_batch(requests, hooks_timeout=None, *, hooks=None, hooks_raise=None)
diff --git a/laya/cli.py b/laya/cli.py
index 89857b4f..0b15b956 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -26,6 +26,7 @@ instead of changing it (it needs an answering mode: routing has no confidence to
"""
import argparse
+import inspect
import json
import sys
@@ -306,6 +307,49 @@ def read_batch_lines(source):
return [line.strip() for line in lines if line.strip()]
+def _takes_predict_diagnostics(router) -> bool:
+ """Whether `router`'s predict_batch takes the per-group sort report of #1111.
+
+ The same signature check the evals runner makes for `sort_by_length`, but exact-name only:
+ the report is a contract with core's own Router, and a `**kwargs` forwarder would swallow
+ the dict while staying as blind as any other foreign router. Routers without the parameter
+ keep the pre-run note, whose file-wide wording is exact for the single group they predict.
+ """
+ fn = getattr(router, "predict_batch", None)
+ if fn is None:
+ return False
+ try:
+ parameters = inspect.signature(fn).parameters
+ except (TypeError, ValueError):
+ return False
+ return "diagnostics" in parameters
+
+
+def _note_sort_report(report, total):
+ """Say what actually reordered, once core has reported it per checkpoint group."""
+ stats = report.get("sort_by_length") or {}
+ groups = stats.get("groups") or 0
+ if not groups or stats.get("reordered", 0) >= groups:
+ return
+ smallest, largest = stats.get("smallest"), stats.get("largest")
+ if groups == 1:
+ # One group saw every line, so the pre-run wording is exact there; keep its copy, with
+ # the group's own size rather than the file's, in case a cache hit shrank the batch.
+ print("note: --sort-by-length only takes effect with --batch-size N where "
+ "1 < N < %d; the states still go in one forward pass" % smallest,
+ file=sys.stderr)
+ return
+ span = "%d" % smallest if smallest == largest else "%d-%d" % (smallest, largest)
+ # The post-run counterpart of the pre-run note: instead of guessing from the file before
+ # the run, name the groups that could not reorder and the size that would have reached them.
+ print("note: --sort-by-length groups within each checkpoint batch, not across the file: "
+ "%d of %d batches (%s states) could not reorder; a batch reorders only when "
+ "--batch-size is smaller than its state count, and the smallest batch holds %d of "
+ "the file's %d lines"
+ % (groups - stats["reordered"], groups, span, smallest, total),
+ file=sys.stderr)
+
+
def run_batch(lines, args, router=None):
"""Route or predict a whole file in one call, so states share checkpoint loads and
forward passes; returns 0 on success, 2 on a handled error."""
@@ -322,19 +366,30 @@ def run_batch(lines, args, router=None):
requests = [{"state": {key: line}, "questions": questions, **overrides}
for line in lines]
kwargs = {"batch_size": args.batch_size, **abstention_override(args)}
+ sort_report = None
if args.sort_by_length:
# Sent only when asked for, as Router.predict_batch itself forwards it: a stub or
# an attached agent that predates the knob keeps working for every run that does
# not request it.
kwargs["sort_by_length"] = True
chunk = args.batch_size if args.batch_size and args.batch_size > 0 else len(lines)
- if not 1 < chunk < len(lines):
+ if _takes_predict_diagnostics(router):
+ # The reorder switches per checkpoint/question group inside core, so the
+ # file's line count is only an upper bound on the denominators that decide
+ # it (#1111): a file that routes to several checkpoints can reorder nothing
+ # while `1 < chunk < len(lines)` holds. Ask core for the per-group outcome
+ # and note after the run, when the groups are known.
+ sort_report = {}
+ kwargs["diagnostics"] = sort_report
+ elif not 1 < chunk < len(lines):
# Core documents this as a no-op rather than an error, so the shell says why
# the run was not reordered instead of printing a silent same-speed result.
print("note: --sort-by-length only takes effect with --batch-size N where "
"1 < N < %d; the states still go in one forward pass" % len(lines),
file=sys.stderr)
results = router.predict_batch(requests, **kwargs)
+ if sort_report is not None:
+ _note_sort_report(sort_report, len(lines))
for line, result in zip(lines, results):
if args.json:
print(json.dumps(result, ensure_ascii=False, default=str))
diff --git a/laya/mcp/remote.py b/laya/mcp/remote.py
index c5f7efc9..9951739b 100644
--- a/laya/mcp/remote.py
+++ b/laya/mcp/remote.py
@@ -203,6 +203,7 @@ class RemoteRouter(Router):
on_predict_start=None,
on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ diagnostics: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
"""One ``/v1/systemone`` call per request, answers in input order.
@@ -210,7 +211,9 @@ class RemoteRouter(Router):
while a Router batch lets each request carry its own questions and controls, so the general
form is the per-request call. ``batch_size`` and ``sort_by_length`` shape forward passes and
have no meaning here; they are accepted and ignored so callers written against ``Router``
- keep working. The server applies its own batching inside each call.
+ keep working, and ``diagnostics`` is the same kind of tolerance: a remote run has no local
+ checkpoint grouping to report (#1111), so the dict is accepted and left untouched.
+ The server applies its own batching inside each call.
"""
results: List[Dict[str, Any]] = []
if hooks_timeout is not None:
diff --git a/laya/router.py b/laya/router.py
index adc69d77..430ef9a6 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -1599,6 +1599,7 @@ class Router(HookRegistry):
on_predict_start=None,
on_predict_end=None,
hooks_raise: Optional[bool] = None,
+ diagnostics: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
"""Route and execute a heterogeneous request batch with minimal model churn.
@@ -1641,6 +1642,13 @@ class Router(HookRegistry):
on_predict_start (PredictHookArg): Plain callable or sequence of callables for start events.
on_predict_end (PredictHookArg): Plain callable or sequence of callables for end events.
hooks_raise: Override the Router's ``hooks_raise`` policy for this call.
+ diagnostics: Optional dict that receives the per-group sort outcome under the key
+ ``"sort_by_length"`` once the batch has run: how many question groups actually
+ reached an agent, how many of those could reorder, and the smallest / largest
+ group size. The reorder switches per group inside this call, so a caller that
+ wants to say whether the knob did anything (#1111) reads this instead of
+ guessing from the request count beforehand. Untouched when ``sort_by_length``
+ is false or no group reached an agent.
Returns:
One normal Router prediction result per request, in the same order as the input.
@@ -1664,6 +1672,11 @@ class Router(HookRegistry):
# `predict` returns as it is.
results: List[Any] = [None] * len(requests)
answered = 0
+ # The per-group sort outcome, kept only for a caller that asked for it (#1111): the
+ # reorder switches per question group inside the loop below, and no caller outside
+ # this method can see the group sizes it forms.
+ sort_report = ({"groups": 0, "reordered": 0, "smallest": None, "largest": None}
+ if sort_by_length and diagnostics is not None else None)
# `compose_hooks`, not `list(self.hooks)`: the composition `predict` uses. It merges
# `set_default_hooks` defaults, then installed hooks, then this call's `hooks`,
# `on_predict_start` and `on_predict_end`. Reading the instance list alone silently
@@ -1783,6 +1796,22 @@ class Router(HookRegistry):
% (len(batch_results), len(items))
)
+ if sort_report is not None:
+ # What this group actually did with the knob: the agent reorders only
+ # when `batch_size` clears its own `1 < chunk < len(states)` gate
+ # (agent.py:1519, onnx_agent.py:761) over this group's states. A group
+ # whose agent predates the knob (#294) reports as unsorted, not as an
+ # error -- the run answered, just at one state's width.
+ n = len(items)
+ reorders = bool("sort_by_length" in batch_kwargs and batch_size
+ and batch_size > 1 and batch_size < n)
+ sort_report["groups"] += 1
+ sort_report["reordered"] += int(reorders)
+ if sort_report["smallest"] is None or n < sort_report["smallest"]:
+ sort_report["smallest"] = n
+ if sort_report["largest"] is None or n > sort_report["largest"]:
+ sort_report["largest"] = n
+
for (i, ctx), result in zip(items, batch_results):
result["routing"] = dict(decisions[i])
ctx.results = [result]
@@ -1821,6 +1850,8 @@ class Router(HookRegistry):
if answered != len(requests):
raise RuntimeError("internal error: batch execution did not produce every result")
+ if sort_report is not None:
+ diagnostics["sort_by_length"] = sort_report
return results
predict_many = predict_batch
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 0abd6059..1994c0cf 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -408,6 +408,68 @@ check("sort: batch routing is untouched by a forward-pass knob",
code == 0 and stub.predict_batch_calls == [] and len(stub.route_batch_calls) == 1,
"predict=%r route=%r" % (stub.predict_batch_calls, len(stub.route_batch_calls)))
+# #1111: the two denominators split when routing groups. Six lines that route to two checkpoints
+# reach `Agent.predict_batch` as groups of 4 and 2 with the same `--batch-size` forwarded to each,
+# so the reorder switches per group while the note used to read the file -- a batch wider than a
+# group reordered nothing, silently. This router splits the way core does and reports the per-group
+# outcome the way core now does, so the post-run note is checkable without checkpoints.
+class GroupingRouter:
+ """One agent-sized call per checkpoint group, plus core's sort report."""
+
+ def __init__(self, sizes):
+ self.sizes = sizes
+ self.calls = []
+
+ def route_batch(self, requests):
+ return [StubDecision() for _ in requests]
+
+ def predict_batch(self, requests, batch_size=None, sort_by_length=False, diagnostics=None):
+ report = {"groups": 0, "reordered": 0, "smallest": None, "largest": None}
+ start = 0
+ for size in self.sizes:
+ n = len(requests[start:start + size])
+ self.calls.append((n, batch_size, sort_by_length))
+ reorders = bool(sort_by_length and batch_size and batch_size > 1 and batch_size < n)
+ report["groups"] += 1
+ report["reordered"] += int(reorders)
+ report["smallest"] = n if report["smallest"] is None else min(report["smallest"], n)
+ report["largest"] = n if report["largest"] is None else max(report["largest"], n)
+ start += size
+ if diagnostics is not None:
+ diagnostics["sort_by_length"] = report
+ return [{"model": "laya-rl", "answers": {"difficulty": {"score": float(i)}}, "usage": {}}
+ for i in range(len(requests))]
+
+
+six = os.path.join(tmp, "six.txt")
+with open(six, "w", encoding="utf-8") as handle:
+ handle.write("one\ntwo\nthree\nfour\nfive\nsix\n")
+
+code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "5",
+ "--sort-by-length"], router=GroupingRouter([4, 2]))
+check("sort: a split batch that reorders nothing is not silent any more",
+ code == 0 and stub.calls == [(4, 5, True), (2, 5, True)]
+ and "2 of 2 batches (2-4 states)" in err,
+ "code %r calls %r err %r" % (code, stub.calls, err))
+check("sort: the per-group note does not quote the file as the denominator",
+ "1 < N < 6" not in err, err)
+
+code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "3",
+ "--sort-by-length"], router=GroupingRouter([4, 2]))
+check("sort: a batch between the group sizes says which half missed out",
+ code == 0 and "1 of 2 batches (2-4 states)" in err, "err %r" % err)
+
+code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "2",
+ "--sort-by-length"], router=GroupingRouter([3, 3]))
+check("sort: a batch below every group reorders everywhere and stays quiet",
+ code == 0 and err == "" and stub.calls == [(3, 2, True), (3, 2, True)],
+ "code %r calls %r err %r" % (code, stub.calls, err))
+
+code, out, err, stub = run_batch_cli(["--batch", six, "--predict", "--batch-size", "6",
+ "--sort-by-length"], router=GroupingRouter([6]))
+check("sort: one group keeps the file-wide wording, sized by the group core saw",
+ code == 0 and "1 < N < 6" in err, "err %r" % err)
+
# A flag nobody can find is a flag nobody uses. README is where `laya --batch` and `/predict/batch`
# are taught, so the same pages that teach `--batch-size` have to teach the grouping that makes a
# bounded pass cheap -- including the condition it needs, since a run without it changes nothing.
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 9b4b0c6d..bebdb456 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -158,6 +158,8 @@ check_param("Agent.predict_batch", Agent.predict_batch, "batch_size", None)
check_param("Agent.predict_batch", Agent.predict_batch, "sort_by_length", False)
check_param("Router.predict_batch", Router.predict_batch, "batch_size", None)
check_param("Router.predict_batch", Router.predict_batch, "sort_by_length", False)
+# the #1111 report: an out-dict the batch fills with the per-group sort outcome
+check_param("Router.predict_batch", Router.predict_batch, "diagnostics", None)
# per-call token-budget overrides
for label, fn in (("Agent.predict_batch", Agent.predict_batch),
diff --git a/tests/test_router.py b/tests/test_router.py
index 6e4efc84..80ee9ef1 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -2065,6 +2065,69 @@ check("near-typo/warnings-as-errors leaves the router unchanged",
(dict(_roll.models), dict(_roll.descriptions), dict(_roll.sha256_digests)), _roll_before)
+# --------------------------------------------------------------------- sort report (#1111)
+# The reorder switches per checkpoint/question group inside predict_batch, on each group's own
+# state count -- a denominator no caller outside can see, which is why the shell's old note read
+# the file and went quiet exactly when auto-routing split a batch into smaller groups. The
+# `diagnostics` out-dict hands the per-group outcome back once the batch has run. Explicit
+# `model` overrides split the batch without loading anything and `load` is stubbed, so this
+# runs with no checkpoints.
+class _SortAgent:
+ def __init__(self, name):
+ self.name = name
+ self.calls = []
+
+ def predict_batch(self, states, questions, batch_size=None, sort_by_length=False):
+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)
+ self.calls.append((len(states), batch_size, sort_by_length,
+ bool(sort_by_length and 1 < chunk < len(states))))
+ return [{"model": self.name, "answers": {}, "usage": {}} for _ in states]
+
+
+_sr = Router()
+_sr_agents = {}
+
+
+def _sr_load(name, _rr=_sr):
+ key = _rr.resolve(name)
+ if key not in _sr_agents:
+ _sr_agents[key] = _SortAgent(key)
+ return _sr_agents[key]
+
+
+_sr.load = _sr_load
+# Six requests like the issue's file: four that route to one checkpoint, two to another.
+_sr_requests = [{"state": {"body": "line %d" % i}, "questions": Q_GENERIC,
+ "model": "english" if i < 4 else "multilingual"} for i in range(6)]
+
+_report = {}
+_sr.predict_batch(_sr_requests, batch_size=5, sort_by_length=True, diagnostics=_report)
+check("sort report/two groups, a batch wider than both reorders neither", _report,
+ {"sort_by_length": {"groups": 2, "reordered": 0, "smallest": 2, "largest": 4}})
+check("sort report/each group still got the knob and kept its own size",
+ [_sr_agents["english"].calls, _sr_agents["multilingual"].calls],
+ [[(4, 5, True, False)], [(2, 5, True, False)]])
+
+for _a in _sr_agents.values():
+ _a.calls = []
+_report = {}
+_sr.predict_batch(_sr_requests, batch_size=3, sort_by_length=True, diagnostics=_report)
+check("sort report/a batch between the group sizes reorders exactly one", _report,
+ {"sort_by_length": {"groups": 2, "reordered": 1, "smallest": 2, "largest": 4}})
+check("sort report/only the group that cleared the gate reordered",
+ [_sr_agents["english"].calls, _sr_agents["multilingual"].calls],
+ [[(4, 3, True, True)], [(2, 3, True, False)]])
+
+_report = {}
+_sr.predict_batch(_sr_requests, batch_size=2, diagnostics=_report)
+check("sort report/untouched when the knob is off", _report, {})
+
+check("sort report/results keep input order regardless",
+ [r["routing"]["model"] for r in _sr.predict_batch(_sr_requests, batch_size=3,
+ sort_by_length=True)],
+ ["english"] * 4 + ["multilingual"] * 2)
+
+
# --------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: