Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(examples): key the cache hook on what the answer depends on
Aashishc3ee5f0bc81e
diff --git a/docs/hooks/examples.md b/docs/hooks/examples.mdindex b4fef349..27e78437 100644--- a/docs/hooks/examples.md+++ b/docs/hooks/examples.md@@ -95,18 +95,21 @@ import layaCACHE = {}-def key(state, questions):- payload = json.dumps([state, questions], sort_keys=True, default=str)+def key(ctx):+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,+ # and the checkpoint and token budget change the answer too.+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ ctx.max_len, ctx.head_max_len], default=str)return hashlib.sha256(payload.encode()).hexdigest()def read(ctx):- hit = CACHE.get(key(ctx.states[0], ctx.questions))+ hit = CACHE.get(key(ctx))if hit is not None:ctx.skip([hit])def write(ctx):if ctx.results:- CACHE[key(ctx.states[0], ctx.questions)] = ctx.results[0]+ CACHE[key(ctx)] = ctx.results[0]agent = laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)first = agent.system_one("state", QUESTIONS) # runs the modeldiff --git a/docs/hooks/patterns.md b/docs/hooks/patterns.mdindex 61e8655d..198779ad 100644--- a/docs/hooks/patterns.md+++ b/docs/hooks/patterns.md@@ -88,17 +88,21 @@ import hashlib, jsonCACHE = {}-def key(state, questions):- return hashlib.sha256(json.dumps([state, questions], sort_keys=True, default=str).encode()).hexdigest()+def key(ctx):+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,+ # and the checkpoint and token budget change the answer too.+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ ctx.max_len, ctx.head_max_len], default=str)+ return hashlib.sha256(payload.encode()).hexdigest()def read(ctx):- hit = CACHE.get(key(ctx.states[0], ctx.questions))+ hit = CACHE.get(key(ctx))if hit is not None:ctx.skip([hit])def write(ctx):if ctx.results:- CACHE[key(ctx.states[0], ctx.questions)] = ctx.results[0]+ CACHE[key(ctx)] = ctx.results[0]laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)```diff --git a/examples/hooks/cache.py b/examples/hooks/cache.pyindex 3c3a0190..4adea990 100644--- a/examples/hooks/cache.py+++ b/examples/hooks/cache.py@@ -1,7 +1,8 @@-"""Cache decisions by state + questions, and skip inference on a hit.+"""Cache decisions, and skip inference on a hit.A start hook calls `ctx.skip(results)`; the engine skips the forward pass and still runs-the end hooks.+the end hooks. The key has to cover everything the answer depends on, so it takes the whole+context rather than just the state and the questions.python examples/hooks/cache.py"""@@ -13,29 +14,46 @@ import layaCACHE = {}-def cache_key(state, questions):- payload = json.dumps({"state": state, "questions": questions}, sort_keys=True, default=str)+def cache_key(ctx):+ """One entry per question the model is actually asked.++ Deliberately not `sort_keys=True`: a choice question's criteria order is positional, so two+ orders are two questions, and `_question_schema` in `laya/router.py` keeps them apart for the+ same reason. Sorting the keys folds them into one entry, and the second caller gets the first+ caller's numbers. `ctx.model` and the token budget belong in the key for the same reason: on+ the Router one hook set serves three checkpoints, and a smaller `max_len` truncates the state.+ """+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,+ ctx.max_len, ctx.head_max_len], default=str)return hashlib.sha256(payload.encode()).hexdigest()def cache_read(ctx):- hit = CACHE.get(cache_key(ctx.states[0], ctx.questions))+ hit = CACHE.get(cache_key(ctx))if hit is not None:ctx.skip([hit])def cache_write(ctx):if ctx.results:- CACHE[cache_key(ctx.states[0], ctx.questions)] = ctx.results[0]+ CACHE[cache_key(ctx)] = ctx.results[0]agent = laya.load("convaiinnovations/laya",on_predict_start=cache_read, on_predict_end=cache_write)STATE = "I was charged twice for the same invoice."-QUESTIONS = {"urgent": {"type": "noul", "instructions": "Is this urgent?"}}+CRITERIA = {"refund": "give me money back", "cancel": "stop the service",+ "other": "anything else"}+QUESTIONS = {"ask": {"type": "choice", "instructions": "What does the customer want?",+ "criteria": dict(CRITERIA)}}+# Same labels and descriptions, criteria written in the other order.+REORDERED = {"ask": {"type": "choice", "instructions": "What does the customer want?",+ "criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],+ "cancel": CRITERIA["cancel"]}}}first = agent.system_one(STATE, QUESTIONS) # runs the model, fills the cache-second = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward pass-print("cache entries:", len(CACHE))-print("same answer:", first["answers"] == second["answers"])+again = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward pass+agent.system_one(STATE, REORDERED) # runs too: a reordered question is a new entry+print("cache entries:", len(CACHE)) # 2, not 1+print("same answer:", first["answers"] == again["answers"])diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex fa750e30..d4e1e640 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -4,10 +4,15 @@ These tests pin the public hook surface (parameter names, kinds, defaults, contelifecycle events, exports) so a change that would break callers fails here first. If a changeis intentional, update this file in the same commit.+The cache key the examples and docs teach is pinned here too: `ctx.skip()` hands back whatever+the key matched, so what a key covers is part of the contract, not an implementation detail.+Run: python tests/test_hooks_api.py"""import dataclasses+import hashlibimport inspect+import jsonimport osimport sys@@ -17,6 +22,7 @@ import laya # noqa: E402from laya import Agent, AsyncHook, BaseHook, PredictContext, PredictHook, Router, load # noqa: E402from laya.hooks import HOOK_EVENTS, Hook # noqa: E402from laya.onnx_agent import ONNXAgent # noqa: E402+from laya.router import _question_schema # noqa: E402PASS, FAIL = [], []@@ -158,6 +164,67 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgentcheck_true("%s/%s exists" % (label, method), callable(getattr(cls, method, None)))+# ------------------------------------------------- the cache key the examples and docs teach+# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md` and+# `docs/hooks/examples.md`, are three copies of one `ctx.skip()` pattern. The key is the part that+# can be wrong: a payload sorted by key folds two criteria orders into one cache entry, while+# `_question_schema` keeps them apart because a choice question's option order is positional. The+# second caller then gets the first caller's answer. Measured on the English checkpoint, a question+# whose criteria were only reordered came back at 0.661 confidence from the cache instead of its+# own 0.496 -- enough to open a gate at 0.6 that a fresh pass would have kept shut.+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))++CRITERIA = {"refund": "give me money back", "cancel": "stop the service", "other": "anything else"}+Q_ORDER_A = {"ask": {"type": "choice", "instructions": "What does the customer want?",+ "criteria": dict(CRITERIA)}}+Q_ORDER_B = {"ask": {"type": "choice", "instructions": "What does the customer want?",+ "criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],+ "cancel": CRITERIA["cancel"]}}}+++def taught_key(path, marker):+ """The key builder of one of those three files, exec'd out of that file's own text.++ Sliced rather than imported: the example calls `laya.load()` at module scope, which would+ download a checkpoint, and a docs code block is not importable at all.+ """+ with open(os.path.join(REPO, path), encoding="utf-8") as handle:+ text = handle.read()+ start = text.index("def %s(" % marker)+ namespace = {"json": json, "hashlib": hashlib}+ exec(text[start:text.index("\ndef ", start + 1)], namespace)+ return namespace[marker]+++def cache_ctx(questions, model="english", max_len=None, head_max_len=None):+ return PredictContext(states=["I was charged twice for the same invoice."],+ questions=questions, model=model, max_len=max_len,+ head_max_len=head_max_len)+++TAUGHT = [("examples/hooks/cache.py", taught_key("examples/hooks/cache.py", "cache_key")),+ ("docs/hooks/patterns.md", taught_key("docs/hooks/patterns.md", "key")),+ ("docs/hooks/examples.md", taught_key("docs/hooks/examples.md", "key"))]++for path, key in TAUGHT:+ ctx = cache_ctx(Q_ORDER_A)+ check_true("%s/reordered criteria is a new entry" % path, key(ctx) != key(cache_ctx(Q_ORDER_B)))+ check_true("%s/reorders are distinct exactly when the core says so" % path,+ (key(ctx) != key(cache_ctx(Q_ORDER_B)))+ == (_question_schema(Q_ORDER_A) != _question_schema(Q_ORDER_B)))+ check_true("%s/the same call is a hit" % path, key(ctx) == key(cache_ctx(Q_ORDER_A)))+ check_true("%s/another checkpoint is a new entry" % path,+ key(ctx) != key(cache_ctx(Q_ORDER_A, model="multilingual")))+ for field in ("max_len", "head_max_len"):+ check_true("%s/another %s is a new entry" % (path, field),+ key(ctx) != key(cache_ctx(Q_ORDER_A, **{field: 256})))++# Three copies, one key: a docs page that drifts from the example fails here, not in someone's+# production cache.+for path, key in TAUGHT[1:]:+ check("%s/keyed like the example" % path, key(cache_ctx(Q_ORDER_A)), TAUGHT[0][1](cache_ctx(Q_ORDER_A)))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL", f)