Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(hooks): make api.md teach the key the example uses
Aashishfbe27d124308
diff --git a/docs/hooks/api.md b/docs/hooks/api.mdindex 50639995..ef32f475 100644--- a/docs/hooks/api.md+++ b/docs/hooks/api.md@@ -64,13 +64,35 @@ class PredictContext:Short-circuits inference. Called from `on_predict_start`, it sets `ctx.results` so the forwardpass is skipped; `on_predict_end` still runs and the supplied results are returned.+The key has to cover everything the answer depends on, and the hook has to cover everything the+call carries: hooks fire once per call, and `predict_batch` calls them with every state at once.+```python+CACHE = {}++def key(ctx, index):+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,+ # and the checkpoint and token budget change the answer too.+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,+ ctx.max_len, ctx.head_max_len], default=str)+ return hashlib.sha256(payload.encode()).hexdigest()+def cache_read(ctx):- hits = [CACHE.get(key(s, ctx.questions)) for s in ctx.states]+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]if all(hit is not None for hit in hits):- ctx.skip(hits) # one entry per state in ctx.states, same shape as predict_batch's return+ ctx.skip(hits) # one entry per state, same shape as predict_batch's return++def cache_write(ctx):+ for i, result in enumerate(ctx.results or []):+ CACHE[key(ctx, i)] = result++laya.load("convaiinnovations/laya", on_predict_start=cache_read, on_predict_end=cache_write)```+`tests/test_hooks_api.py` execs this block, `examples/hooks/cache.py`, and the caching blocks of+`docs/hooks/patterns.md` and `docs/hooks/examples.md`, and asserts all four keys the same way, so a+page cannot teach a key the example has moved on from.+On the `Router`, a skipped payload gets a `routing` key added (without overwriting one italready has), so `Router.predict` keeps its documented return shape.diff --git a/examples/hooks/cache.py b/examples/hooks/cache.pyindex 3aa1d83e..147cabc0 100644--- a/examples/hooks/cache.py+++ b/examples/hooks/cache.py@@ -13,6 +13,7 @@ import jsonimport layaCACHE = {}+SKIPS = [] # one entry per call `cache_read` served, so the prints below are checkabledef cache_key(ctx, index):@@ -32,6 +33,7 @@ def cache_key(ctx, index):def cache_read(ctx):hits = [CACHE.get(cache_key(ctx, i)) for i in range(len(ctx.states))]if all(hit is not None for hit in hits):+ SKIPS.append(len(hits))ctx.skip(hits) # one per state: `skip` replaces the whole call, not its first result@@ -60,8 +62,12 @@ agent.system_one(STATE, REORDERED) # runs too: a reordered questionprint("cache entries:", len(CACHE)) # 2, not 1print("same answer:", first["answers"] == again["answers"])+# Warm the second state before the batch. `cache_read` skips only when *every* state of the call+# has an entry, so a batch that mixes one warm state with one cold state runs a full forward.+agent.system_one(OTHER, QUESTIONS)# Batched, in the other order: both states are cached, so the whole call is served.served = agent.predict_batch([OTHER, STATE], QUESTIONS)print("results for a 2-state batch:", len(served))print("one answer per state, in the caller's order:",[r["answers"]["ask"]["choice"] for r in served])+print("calls served without a forward pass:", len(SKIPS)) # 2: `again`, and this batchdiff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex a8dd0419..bf457866 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -9,10 +9,12 @@ the key matched, so what a key covers is part of the contract, not an implementaRun: python tests/test_hooks_api.py"""+import contextlibimport dataclassesimport functoolsimport hashlibimport inspect+import ioimport jsonimport osimport sys@@ -166,8 +168,9 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgent# ------------------------------------------------- the cache key the examples and docs teach-# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md` and-# `docs/hooks/examples.md`, are three copies of one `ctx.skip()` pattern. The key is the part that+# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md`,+# `docs/hooks/examples.md` and `docs/hooks/api.md`, are four copies of one `ctx.skip()` pattern.+# The key is the part that# can be wrong: a payload sorted by key folds two criteria orders into one cache entry, while# `_question_schema` keeps them apart because a choice question's option order is positional. The# second caller then gets the first caller's answer. Measured on the English checkpoint, a question@@ -191,13 +194,14 @@ def taught_cache(path):Sliced rather than imported: the example calls `laya.load()` at module scope, which woulddownload a checkpoint, and a docs code block is not importable at all. The slice stops at that- `laya.load(...)` line for the same reason, so none of this needs weights.+ `laya.load(...)` line, or at the closing fence of the block, for the same reason, so none of+ this needs weights."""with open(os.path.join(REPO, path), encoding="utf-8") as handle:text = handle.read()start = text.index("CACHE = {")stop = len(text)- for cut in ("\nlaya.load", "\nagent = laya.load"):+ for cut in ("\nlaya.load", "\nagent = laya.load", "\n```"):found = text.find(cut, start)if found != -1:stop = min(stop, found)@@ -226,7 +230,10 @@ def answers_for(states):TAUGHT = [("examples/hooks/cache.py", taught_cache("examples/hooks/cache.py")),("docs/hooks/patterns.md", taught_cache("docs/hooks/patterns.md")),- ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md"))]+ ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md")),+ # `api.md` is the page that documents `ctx.skip()`, so it is the page a reader copies+ # from -- and it carries the same key, which makes it the copy most able to go stale.+ ("docs/hooks/api.md", taught_cache("docs/hooks/api.md"))]for path, namespace in TAUGHT:key = functools.partial(key_of, namespace)@@ -285,6 +292,67 @@ for path, namespace in TAUGHT[1:]:key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A, states=BATCH), 1))+# ------------------------------------------- what the cache example's own demo claims+# The example closes by saying its two-state batch "is served", and nothing in its printed output+# distinguishes a served payload from a fresh one -- both are the same answers. So run the demo+# against a stub agent that counts its own forward passes, and check the claim rather than trusting+# the comment. No weights: `laya.load` is the only thing the example takes from the library.+def run_cache_example():+ with open(os.path.join(REPO, "examples/hooks/cache.py"), encoding="utf-8") as handle:+ source = handle.read().replace("\nimport laya\n", "\n")+ forwards = []++ class StubAgent:+ def __init__(self, on_start, on_end):+ self.on_start, self.on_end = on_start, on_end++ def _call(self, states, questions):+ ctx = PredictContext(states=list(states), questions=questions, model="english")+ if self.on_start:+ self.on_start(ctx)+ if ctx.results is None:+ forwards.append(len(states))+ ctx.results = [{"model": "laya-rl-agent",+ "answers": {"ask": {"type": "choice", "choice": s[:8]}}}+ for s in states]+ ctx.usage = {"input_tokens": 10 * len(states), "output_tokens": 0}+ if self.on_end:+ self.on_end(ctx)+ return ctx.results++ def system_one(self, state, questions, model=None):+ return self._call([state], questions)[0]++ def predict_batch(self, states, questions, model=None, batch_size=None):+ return self._call(states, questions)++ class StubLaya:+ @staticmethod+ def load(name, on_predict_start=None, on_predict_end=None):+ return StubAgent(on_predict_start, on_predict_end)++ out = io.StringIO()+ namespace = {"laya": StubLaya}+ with contextlib.redirect_stdout(out):+ exec(compile(source, "examples/hooks/cache.py", "exec"), namespace)+ return namespace, forwards, out.getvalue()+++example, example_forwards, example_out = run_cache_example()+# Why the exact list: the repeat of STATE and the warm batch are the two served calls, and the+# batch is the only multi-state one. A demo that mixed a cold state into the batch would show+# `[1]` here, which is the claim its comment does not make.+check("examples/hooks/cache.py/the repeat and the batch are the served calls",+ example["SKIPS"], [1, 2])+check("examples/hooks/cache.py/one forward per new entry, none for the batch",+ example_forwards, [1, 1, 1])+check_true("examples/hooks/cache.py/its own output still says two entries",+ "cache entries: 2" in example_out, example_out)+check("examples/hooks/cache.py/the batch answers in the caller's order",+ [r["answers"]["ask"]["choice"] for r in example["served"]],+ [example["OTHER"][:8], example["STATE"][:8]])++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL", f)