Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(examples): let the cache hook cover every state the call carries
Aashish8b332fe0c00c
diff --git a/docs/hooks/api.md b/docs/hooks/api.mdindex 8817e666..50639995 100644--- a/docs/hooks/api.md+++ b/docs/hooks/api.md@@ -66,9 +66,9 @@ pass is skipped; `on_predict_end` still runs and the supplied results are return```pythondef cache_read(ctx):- hit = CACHE.get(key(ctx.states[0], ctx.questions))- if hit is not None:- ctx.skip([hit]) # list of per-state results, same shape as predict_batch's return+ hits = [CACHE.get(key(s, ctx.questions)) for s in ctx.states]+ if all(hit is not None for hit in hits):+ ctx.skip(hits) # one entry per state in ctx.states, same shape as predict_batch's return```On the `Router`, a skipped payload gets a `routing` key added (without overwriting one itdiff --git a/docs/hooks/examples.md b/docs/hooks/examples.mdindex 27e78437..66bb6852 100644--- a/docs/hooks/examples.md+++ b/docs/hooks/examples.md@@ -95,21 +95,21 @@ import layaCACHE = {}-def key(ctx):+def key(ctx, index):# Not sort_keys=True: criteria order is positional, so two orders are two questions,# and the checkpoint and token budget change the answer too.- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,ctx.max_len, ctx.head_max_len], default=str)return hashlib.sha256(payload.encode()).hexdigest()def read(ctx):- hit = CACHE.get(key(ctx))- if hit is not None:- ctx.skip([hit])+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]+ if all(hit is not None for hit in hits):+ ctx.skip(hits) # one per state: skip replaces the whole calldef write(ctx):- if ctx.results:- CACHE[key(ctx)] = ctx.results[0]+ for i, result in enumerate(ctx.results or []):+ CACHE[key(ctx, i)] = resultagent = laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)first = agent.system_one("state", QUESTIONS) # runs the modeldiff --git a/docs/hooks/patterns.md b/docs/hooks/patterns.mdindex 198779ad..344d9d54 100644--- a/docs/hooks/patterns.md+++ b/docs/hooks/patterns.md@@ -88,27 +88,29 @@ import hashlib, jsonCACHE = {}-def key(ctx):+def key(ctx, index):# Not sort_keys=True: criteria order is positional, so two orders are two questions,# and the checkpoint and token budget change the answer too.- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,ctx.max_len, ctx.head_max_len], default=str)return hashlib.sha256(payload.encode()).hexdigest()def read(ctx):- hit = CACHE.get(key(ctx))- if hit is not None:- ctx.skip([hit])+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]+ if all(hit is not None for hit in hits):+ ctx.skip(hits) # one per state: skip replaces the whole calldef write(ctx):- if ctx.results:- CACHE[key(ctx)] = ctx.results[0]+ for i, result in enumerate(ctx.results or []):+ CACHE[key(ctx, i)] = resultlaya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)```-Guard the cache with a lock when serving concurrently. On the Router the cached payload still-gets a `routing` key, so the return shape is unchanged.+Hooks fire once per call, so keying on one state is not enough on `predict_batch`: `ctx.skip()`+replaces every result the call would have returned. Guard the cache with a lock when serving+concurrently. On the Router the cached payload still gets a `routing` key, so the return shape is+unchanged.### Metricsdiff --git a/examples/hooks/cache.py b/examples/hooks/cache.pyindex 4adea990..3aa1d83e 100644--- a/examples/hooks/cache.py+++ b/examples/hooks/cache.py@@ -1,8 +1,9 @@"""Cache decisions, and skip inference on a hit.A start hook calls `ctx.skip(results)`; the engine skips the forward pass and still runs-the end hooks. The key has to cover everything the answer depends on, so it takes the whole-context rather than just the state and the questions.+the end hooks. The key has to cover everything the answer depends on, and a hook has to cover+everything the call carries: hooks fire once per call, and `predict_batch` calls them with+every state of the batch at once.python examples/hooks/cache.py"""@@ -14,8 +15,8 @@ import layaCACHE = {}-def cache_key(ctx):- """One entry per question the model is actually asked.+def cache_key(ctx, index):+ """One entry per question the model is actually asked, for one state of the call.Deliberately not `sort_keys=True`: a choice question's criteria order is positional, so twoorders are two questions, and `_question_schema` in `laya/router.py` keeps them apart for the@@ -23,26 +24,27 @@ def cache_key(ctx):caller's numbers. `ctx.model` and the token budget belong in the key for the same reason: onthe Router one hook set serves three checkpoints, and a smaller `max_len` truncates the state."""- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,ctx.max_len, ctx.head_max_len], default=str)return hashlib.sha256(payload.encode()).hexdigest()def cache_read(ctx):- hit = CACHE.get(cache_key(ctx))- if hit is not None:- ctx.skip([hit])+ hits = [CACHE.get(cache_key(ctx, i)) for i in range(len(ctx.states))]+ if all(hit is not None for hit in hits):+ ctx.skip(hits) # one per state: `skip` replaces the whole call, not its first resultdef cache_write(ctx):- if ctx.results:- CACHE[cache_key(ctx)] = ctx.results[0]+ for i, result in enumerate(ctx.results or []):+ CACHE[cache_key(ctx, i)] = resultagent = laya.load("convaiinnovations/laya",on_predict_start=cache_read, on_predict_end=cache_write)STATE = "I was charged twice for the same invoice."+OTHER = "Where do I change my notification settings?"CRITERIA = {"refund": "give me money back", "cancel": "stop the service","other": "anything else"}QUESTIONS = {"ask": {"type": "choice", "instructions": "What does the customer want?",@@ -57,3 +59,9 @@ again = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward pagent.system_one(STATE, REORDERED) # runs too: a reordered question is a new entryprint("cache entries:", len(CACHE)) # 2, not 1print("same answer:", first["answers"] == again["answers"])++# Batched, in the other order: both states are cached, so the whole call is served.+served = agent.predict_batch([OTHER, STATE], QUESTIONS)+print("results for a 2-state batch:", len(served))+print("one answer per state, in the caller's order:",+ [r["answers"]["ask"]["choice"] for r in served])diff --git a/laya/hooks.py b/laya/hooks.pyindex 58059b8d..b3afbd87 100644--- a/laya/hooks.py+++ b/laya/hooks.py@@ -46,7 +46,12 @@ class PredictContext:error: Optional[BaseException] = Nonedef skip(self, results: List[Dict[str, Any]]) -> None:- """Set cached results from a start hook; inference is skipped, end hooks still run."""+ """Set cached results from a start hook; inference is skipped, end hooks still run.++ `results` replaces the whole call, so it carries one entry per state in `ctx.states` --+ the shape `predict_batch` returns -- in that order. A hook fires once per call, and a+ call can carry many states.+ """self.results = resultsdiff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex d4e1e640..a8dd0419 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -10,6 +10,7 @@ the key matched, so what a key covers is part of the contract, not an implementaRun: python tests/test_hooks_api.py"""import dataclasses+import functoolsimport hashlibimport inspectimport json@@ -180,33 +181,57 @@ Q_ORDER_A = {"ask": {"type": "choice", "instructions": "What does the customer wQ_ORDER_B = {"ask": {"type": "choice", "instructions": "What does the customer want?","criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],"cancel": CRITERIA["cancel"]}}}+BATCH = ["I was charged twice for the same invoice.",+ "The app crashes every time I open the export screen.",+ "Where do I change my notification settings?"]-def taught_key(path, marker):- """The key builder of one of those three files, exec'd out of that file's own text.+def taught_cache(path):+ """One taught copy of the cache pattern, exec'd out of that file's own text.Sliced rather than imported: the example calls `laya.load()` at module scope, which would- download a checkpoint, and a docs code block is not importable at all.+ download a checkpoint, and a docs code block is not importable at all. The slice stops at that+ `laya.load(...)` line for the same reason, so none of this needs weights."""with open(os.path.join(REPO, path), encoding="utf-8") as handle:text = handle.read()- start = text.index("def %s(" % marker)- namespace = {"json": json, "hashlib": hashlib}- exec(text[start:text.index("\ndef ", start + 1)], namespace)- return namespace[marker]+ start = text.index("CACHE = {")+ stop = len(text)+ for cut in ("\nlaya.load", "\nagent = laya.load"):+ found = text.find(cut, start)+ if found != -1:+ stop = min(stop, found)+ namespace = {"json": json, "hashlib": hashlib, "laya": laya}+ exec(text[start:stop], namespace)+ return namespace-def cache_ctx(questions, model="english", max_len=None, head_max_len=None):- return PredictContext(states=["I was charged twice for the same invoice."],- questions=questions, model=model, max_len=max_len,- head_max_len=head_max_len)+def key_of(namespace, ctx, index=0):+ """The copy's key builder, called the way that copy declares it."""+ key = namespace.get("cache_key") or namespace["key"]+ return key(ctx, index) if len(sig(key)) > 1 else key(ctx)-TAUGHT = [("examples/hooks/cache.py", taught_key("examples/hooks/cache.py", "cache_key")),- ("docs/hooks/patterns.md", taught_key("docs/hooks/patterns.md", "key")),- ("docs/hooks/examples.md", taught_key("docs/hooks/examples.md", "key"))]+def cache_ctx(questions, model="english", max_len=None, head_max_len=None, states=(BATCH[0],)):+ return PredictContext(states=list(states), questions=questions, model=model,+ max_len=max_len, head_max_len=head_max_len)-for path, key in TAUGHT:++def answers_for(states):+ """Per-state payloads, each labelled with its own state, so a mix-up is visible."""+ return [{"model": "laya-rl-agent", "answers": {"ask": {"type": "choice", "choice": tag}},+ "usage": {"input_tokens": 10 + i}} for i, tag in enumerate(+ ["billing", "support", "other", "refund", "cancel"][:len(states)])]+++TAUGHT = [("examples/hooks/cache.py", taught_cache("examples/hooks/cache.py")),+ ("docs/hooks/patterns.md", taught_cache("docs/hooks/patterns.md")),+ ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md"))]++for path, namespace in TAUGHT:+ key = functools.partial(key_of, namespace)+ read = namespace.get("cache_read") or namespace["read"]+ write = namespace.get("cache_write") or namespace["write"]ctx = cache_ctx(Q_ORDER_A)check_true("%s/reordered criteria is a new entry" % path, key(ctx) != key(cache_ctx(Q_ORDER_B)))check_true("%s/reorders are distinct exactly when the core says so" % path,@@ -219,10 +244,45 @@ for path, key in TAUGHT:check_true("%s/another %s is a new entry" % (path, field),key(ctx) != key(cache_ctx(Q_ORDER_A, **{field: 256})))+ # A hook fires once per call, and a call can carry a whole batch.+ check("%s/key takes the state it describes" % path,+ len(sig(namespace.get("cache_key") or namespace["key"])), 2)+ multi = cache_ctx(Q_ORDER_A, states=BATCH)+ check_true("%s/two states of one call are two entries" % path,+ key(multi, 0) != key(multi, 1))+ namespace["CACHE"].clear()+ filled = cache_ctx(Q_ORDER_A, states=BATCH)+ filled.results = answers_for(BATCH)+ write(filled)+ check("%s/write stores one entry per state" % path, len(namespace["CACHE"]), len(BATCH))+ warm = cache_ctx(Q_ORDER_A, states=list(reversed(BATCH)))+ read(warm)+ check("%s/a warm batch is served back per state, in the caller's order" % path,+ warm.results, list(reversed(filled.results)))+ cold = cache_ctx(Q_ORDER_A, states=BATCH[:2] + ["an uncached state"])+ read(cold)+ check_true("%s/a batch with one uncached state still runs" % path, cold.results is None)+ single = cache_ctx(Q_ORDER_A, states=[BATCH[1]])+ read(single)+ check("%s/a single-state call is one entry" % path, single.results, [filled.results[1]])+ empty = cache_ctx(Q_ORDER_A, states=[])+ try:+ read(empty)+ served = empty.results+ except Exception as exc: # a hook that raises on an empty call is a failure+ served = "raised %s" % exc.__class__.__name__+ check("%s/an empty batch skips to an empty list" % path, served, [])+ namespace["CACHE"].clear()+# Three copies, one key: a docs page that drifts from the example fails here, not in someone's# production cache.-for path, key in TAUGHT[1:]:- check("%s/keyed like the example" % path, key(cache_ctx(Q_ORDER_A)), TAUGHT[0][1](cache_ctx(Q_ORDER_A)))+for path, namespace in TAUGHT[1:]:+ check("%s/keyed like the example" % path,+ key_of(namespace, cache_ctx(Q_ORDER_A)),+ key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A)))+ check("%s/per-state like the example" % path,+ key_of(namespace, cache_ctx(Q_ORDER_A, states=BATCH), 1),+ key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A, states=BATCH), 1))print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))