Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #654 from aashish254/feat-batch-sort-by-length

Nandakishorbb931aae8050
diff --cc README.md
index 3872e9dd,9ec3b1aa..1bc815a7
--- a/README.md
+++ b/README.md
@@@ -271,36 -284,16 +284,46 @@@ curl -s localhost:8000/predict -H 'cont
`--no-preload` loads checkpoints lazily instead of all three up front; `--device cuda|cpu|mps`
pins the device. See `python examples/server.py --help` for the rest.
+A request body may carry `model` to pin a checkpoint instead of letting the router choose, and it
+takes exactly the spellings `laya --model` takes: names, aliases and casing all resolve through
+`laya.router`. `GET /models` lists the checkpoints and the aliases alongside them.
+
+ `/predict/batch` accepts two optional body fields that control the shape of the forward passes
+ without changing any answer: `batch_size` (states per pass; omit it and the whole batch is one
+ pass) and `sort_by_length` (group similarly sized states so each pass pads to a shorter maximum —
+ it needs a `batch_size` below the number of states, since one pass has nothing to reorder).
+ Measured on the running server: 64 real reviews of 91–2,293 characters, one choice question,
+ `batch_size: 8`, three interleaved rounds on MPS — the default one-pass shape 5.35s median against
+ 3.04s sorted, a paired median of **1.77x** (min 1.76x), and 64/64 labels identical in input order.
+ `batch_size: 8` on its own was 4.95s, so the length grouping is what earns the second factor:
+ 1.66x over the sized batch.
+
+### JavaScript / TypeScript
+
+[`laya-client`](sdk/typescript/README.md) supports JavaScript and TypeScript
+applications over HTTP to a self-hosted `laya-serve` `/v1/systemone` server. It
+includes inferred answer types, all five question presets, ESM/CommonJS exports,
+and cancellation.
+
+```bash
+# From this checkout, install and start the Python inference service:
+pip install -e '.[serve]'
+LAYA_HOST=127.0.0.1 LAYA_MODELS=english laya-serve
+
+# In another terminal:
+cd sdk/typescript
+npm ci
+npm run build
+node examples/triage.mjs
+```
+
+The npm package is named `laya-client`. Use it when a JavaScript or TypeScript
+application talks over HTTP to self-hosted Python `laya-serve`. Use `laya-ts`
+when inference must run directly inside JavaScript through its local ONNX
+runtime, without a Python server.
+See [installation, examples, and publishing](sdk/typescript/README.md) and the
+[repository analysis](docs/typescript-sdk.md) for architecture and scope.
+
---
## Quickstart: Route Mode (Recommended)
diff --cc examples/server.py
index 5f17c414,0ce466e2..96c68a20
--- a/examples/server.py
+++ b/examples/server.py
@@@ -207,10 -151,19 +207,20 @@@ class PredictRequest(BaseModel)
class BatchRequest(BaseModel):
states: List[Union[str, Dict[str, Any], List[Any]]] = Field(..., min_length=1, max_length=64)
questions: Dict[str, Question] = Field(..., min_length=1)
- model: Optional[str] = None
+ model: Optional[str] = Field(default=None, description="Checkpoint override: "
+ + _model_names())
task: Optional[str] = None
lang: Optional[str] = None
+ # The batch shape, not a per-state control: these size and group the forward passes rather
+ # than saying what a state means. `sort_by_length` only reorders a batch that is split into
+ # more than one pass, so it needs a `batch_size` strictly between 1 and the state count.
+ batch_size: Optional[int] = Field(
+ default=None, ge=1,
+ description="states per forward pass; the default sends the whole batch in one pass")
+ sort_by_length: bool = Field(
+ default=False,
+ description="group similarly sized states in the same pass so each pads to a shorter "
+ "maximum; needs a batch_size below the state count, never changes an answer")
_v_model = field_validator("model")(classmethod(lambda cls, v: _check_model(v)))
diff --cc tests/test_evals_api.py
index 6a08a780,f20eb2a8..26742062
--- a/tests/test_evals_api.py
+++ b/tests/test_evals_api.py
@@@ -39,16 -40,53 +40,53 @@@ check("default evaluator names"
check_true("ScoreWithin names its tolerance", evals.ScoreWithin(0.25).name == "score_within_0.25")
# --------------------------------------------------------------- exports / callables
-for name in ("Dataset", "Example", "EvalError", "EvalReport", "evaluate", "ece", "assert_regression"):
+for name in ("Dataset", "Example", "EvalError", "EvalReport", "evaluate", "ece", "assert_regression",
+ "REPORT_SCHEMA", "questions_fingerprint", "file_fingerprint"):
check_true("laya.evals.%s exists" % name, hasattr(evals, name))
+check("REPORT_SCHEMA", evals.REPORT_SCHEMA, "laya-evals-report/1")
+check_true("EvalReport.comparable_to is part of the report contract",
+ callable(getattr(evals.EvalReport, "comparable_to", None)))
+check("EvalReport fields are unchanged", [f.name for f in dataclasses.fields(evals.EvalReport)],
+ ["config", "overall", "slices", "cases"])
+
+ # --------------------------------------------------------------- evaluate's call signature
+ # `evaluate` is the harness entry point callers extend from their own scripts, so its parameter
+ # names and order are the contract. `sort_by_length` is core's #294 forward-pass grouping knob
+ # reaching a scored run through `laya-evals run --sort-by-length`; it is appended, never inserted,
+ # because a caller that passed `on_error` or `config` positionally must still mean the same thing.
+ _sig = inspect.signature(evals.evaluate)
+ check("evaluate signature", [p.name for p in _sig.parameters.values()],
+ ["runner", "dataset", "evaluators", "batch_size", "on_error", "config", "sort_by_length"])
+ check("evaluate: grouping defaults to off", _sig.parameters["sort_by_length"].default, False)
+ check_true("evaluate: the knob sits after the batch size it groups inside",
+ list(_sig.parameters).index("batch_size")
+ < list(_sig.parameters).index("sort_by_length"))
+
+ # The two batch entry points the CLI wires up: both have to take the knob by the same name, or a
+ # `--sort-by-length` run reports `sort_by_length_sent: false` for the surface it ships.
+ from laya import evals_cli # noqa: E402
+
+ for label, fn in (("RouterRunner.predict_batch", evals_cli.RouterRunner.predict_batch),
+ ("OnnxRunner.predict_batch", evals_cli.OnnxRunner.predict_batch)):
+ params = inspect.signature(fn).parameters
+ check_true("%s takes the grouping knob" % label, "sort_by_length" in params)
+ check("%s defaults it to off" % label, params["sort_by_length"].default, False)
+
+ # A signature is not a wiring: a runner that declares the knob and drops it on the way through
+ # would pass both checks above. `OnnxRunner`'s call is gated in test_evals_onnx.py against a
+ # recording agent; this is the same witness for the router-shaped runner the CLI uses by default.
+ class _RecordingRouter:
+ def __init__(self):
+ self.calls = []
+
+ def predict_batch(self, requests, batch_size=None, sort_by_length=False):
+ self.calls.append({"batch_size": batch_size, "sort_by_length": sort_by_length})
+ return [{"model": "m", "answers": {}} for _ in requests]
+
+
+ _router = _RecordingRouter()
-_runner = evals_cli.RouterRunner(_router)
-_runner.predict_batch(["s1", "s2"], {"intent": {"type": "choice"}}, batch_size=2, sort_by_length=True)
-check("RouterRunner hands the grouping to the router it wraps",
- _router.calls[-1], {"batch_size": 2, "sort_by_length": True})
-_runner.predict_batch(["s1", "s2"], {"intent": {"type": "choice"}}, batch_size=2)
-check("RouterRunner does not invent it for a run that did not ask",
- _router.calls[-1], {"batch_size": 2, "sort_by_length": False})
+
# --------------------------------------------------------------- torch stays out of import
probe = subprocess.run(
[sys.executable, "-c",