Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(examples): key the cache hook on what the answer depends on

Aashishc3ee5f0bc81e
diff --git a/docs/hooks/examples.md b/docs/hooks/examples.md
index b4fef349..27e78437 100644
--- a/docs/hooks/examples.md
+++ b/docs/hooks/examples.md
@@ -95,18 +95,21 @@ import laya
CACHE = {}
-def key(state, questions):
- payload = json.dumps([state, questions], sort_keys=True, default=str)
+def key(ctx):
+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,
+ # and the checkpoint and token budget change the answer too.
+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ ctx.max_len, ctx.head_max_len], default=str)
return hashlib.sha256(payload.encode()).hexdigest()
def read(ctx):
- hit = CACHE.get(key(ctx.states[0], ctx.questions))
+ hit = CACHE.get(key(ctx))
if hit is not None:
ctx.skip([hit])
def write(ctx):
if ctx.results:
- CACHE[key(ctx.states[0], ctx.questions)] = ctx.results[0]
+ CACHE[key(ctx)] = ctx.results[0]
agent = laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)
first = agent.system_one("state", QUESTIONS) # runs the model
diff --git a/docs/hooks/patterns.md b/docs/hooks/patterns.md
index 61e8655d..198779ad 100644
--- a/docs/hooks/patterns.md
+++ b/docs/hooks/patterns.md
@@ -88,17 +88,21 @@ import hashlib, json
CACHE = {}
-def key(state, questions):
- return hashlib.sha256(json.dumps([state, questions], sort_keys=True, default=str).encode()).hexdigest()
+def key(ctx):
+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,
+ # and the checkpoint and token budget change the answer too.
+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ ctx.max_len, ctx.head_max_len], default=str)
+ return hashlib.sha256(payload.encode()).hexdigest()
def read(ctx):
- hit = CACHE.get(key(ctx.states[0], ctx.questions))
+ hit = CACHE.get(key(ctx))
if hit is not None:
ctx.skip([hit])
def write(ctx):
if ctx.results:
- CACHE[key(ctx.states[0], ctx.questions)] = ctx.results[0]
+ CACHE[key(ctx)] = ctx.results[0]
laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)
```
diff --git a/examples/hooks/cache.py b/examples/hooks/cache.py
index 3c3a0190..4adea990 100644
--- a/examples/hooks/cache.py
+++ b/examples/hooks/cache.py
@@ -1,7 +1,8 @@
-"""Cache decisions by state + questions, and skip inference on a hit.
+"""Cache decisions, and skip inference on a hit.
A start hook calls `ctx.skip(results)`; the engine skips the forward pass and still runs
-the end hooks.
+the end hooks. The key has to cover everything the answer depends on, so it takes the whole
+context rather than just the state and the questions.
python examples/hooks/cache.py
"""
@@ -13,29 +14,46 @@ import laya
CACHE = {}
-def cache_key(state, questions):
- payload = json.dumps({"state": state, "questions": questions}, sort_keys=True, default=str)
+def cache_key(ctx):
+ """One entry per question the model is actually asked.
+
+ Deliberately not `sort_keys=True`: a choice question's criteria order is positional, so two
+ orders are two questions, and `_question_schema` in `laya/router.py` keeps them apart for the
+ same reason. Sorting the keys folds them into one entry, and the second caller gets the first
+ caller's numbers. `ctx.model` and the token budget belong in the key for the same reason: on
+ the Router one hook set serves three checkpoints, and a smaller `max_len` truncates the state.
+ """
+ payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ ctx.max_len, ctx.head_max_len], default=str)
return hashlib.sha256(payload.encode()).hexdigest()
def cache_read(ctx):
- hit = CACHE.get(cache_key(ctx.states[0], ctx.questions))
+ hit = CACHE.get(cache_key(ctx))
if hit is not None:
ctx.skip([hit])
def cache_write(ctx):
if ctx.results:
- CACHE[cache_key(ctx.states[0], ctx.questions)] = ctx.results[0]
+ CACHE[cache_key(ctx)] = ctx.results[0]
agent = laya.load("convaiinnovations/laya",
on_predict_start=cache_read, on_predict_end=cache_write)
STATE = "I was charged twice for the same invoice."
-QUESTIONS = {"urgent": {"type": "noul", "instructions": "Is this urgent?"}}
+CRITERIA = {"refund": "give me money back", "cancel": "stop the service",
+ "other": "anything else"}
+QUESTIONS = {"ask": {"type": "choice", "instructions": "What does the customer want?",
+ "criteria": dict(CRITERIA)}}
+# Same labels and descriptions, criteria written in the other order.
+REORDERED = {"ask": {"type": "choice", "instructions": "What does the customer want?",
+ "criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],
+ "cancel": CRITERIA["cancel"]}}}
first = agent.system_one(STATE, QUESTIONS) # runs the model, fills the cache
-second = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward pass
-print("cache entries:", len(CACHE))
-print("same answer:", first["answers"] == second["answers"])
+again = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward pass
+agent.system_one(STATE, REORDERED) # runs too: a reordered question is a new entry
+print("cache entries:", len(CACHE)) # 2, not 1
+print("same answer:", first["answers"] == again["answers"])
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index fa750e30..d4e1e640 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -4,10 +4,15 @@ These tests pin the public hook surface (parameter names, kinds, defaults, conte
lifecycle events, exports) so a change that would break callers fails here first. If a change
is intentional, update this file in the same commit.
+The cache key the examples and docs teach is pinned here too: `ctx.skip()` hands back whatever
+the key matched, so what a key covers is part of the contract, not an implementation detail.
+
Run: python tests/test_hooks_api.py
"""
import dataclasses
+import hashlib
import inspect
+import json
import os
import sys
@@ -17,6 +22,7 @@ import laya # noqa: E402
from laya import Agent, AsyncHook, BaseHook, PredictContext, PredictHook, Router, load # noqa: E402
from laya.hooks import HOOK_EVENTS, Hook # noqa: E402
from laya.onnx_agent import ONNXAgent # noqa: E402
+from laya.router import _question_schema # noqa: E402
PASS, FAIL = [], []
@@ -158,6 +164,67 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgent
check_true("%s/%s exists" % (label, method), callable(getattr(cls, method, None)))
+# ------------------------------------------------- the cache key the examples and docs teach
+# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md` and
+# `docs/hooks/examples.md`, are three copies of one `ctx.skip()` pattern. The key is the part that
+# can be wrong: a payload sorted by key folds two criteria orders into one cache entry, while
+# `_question_schema` keeps them apart because a choice question's option order is positional. The
+# second caller then gets the first caller's answer. Measured on the English checkpoint, a question
+# whose criteria were only reordered came back at 0.661 confidence from the cache instead of its
+# own 0.496 -- enough to open a gate at 0.6 that a fresh pass would have kept shut.
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+CRITERIA = {"refund": "give me money back", "cancel": "stop the service", "other": "anything else"}
+Q_ORDER_A = {"ask": {"type": "choice", "instructions": "What does the customer want?",
+ "criteria": dict(CRITERIA)}}
+Q_ORDER_B = {"ask": {"type": "choice", "instructions": "What does the customer want?",
+ "criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],
+ "cancel": CRITERIA["cancel"]}}}
+
+
+def taught_key(path, marker):
+ """The key builder of one of those three files, exec'd out of that file's own text.
+
+ Sliced rather than imported: the example calls `laya.load()` at module scope, which would
+ download a checkpoint, and a docs code block is not importable at all.
+ """
+ with open(os.path.join(REPO, path), encoding="utf-8") as handle:
+ text = handle.read()
+ start = text.index("def %s(" % marker)
+ namespace = {"json": json, "hashlib": hashlib}
+ exec(text[start:text.index("\ndef ", start + 1)], namespace)
+ return namespace[marker]
+
+
+def cache_ctx(questions, model="english", max_len=None, head_max_len=None):
+ return PredictContext(states=["I was charged twice for the same invoice."],
+ questions=questions, model=model, max_len=max_len,
+ head_max_len=head_max_len)
+
+
+TAUGHT = [("examples/hooks/cache.py", taught_key("examples/hooks/cache.py", "cache_key")),
+ ("docs/hooks/patterns.md", taught_key("docs/hooks/patterns.md", "key")),
+ ("docs/hooks/examples.md", taught_key("docs/hooks/examples.md", "key"))]
+
+for path, key in TAUGHT:
+ ctx = cache_ctx(Q_ORDER_A)
+ check_true("%s/reordered criteria is a new entry" % path, key(ctx) != key(cache_ctx(Q_ORDER_B)))
+ check_true("%s/reorders are distinct exactly when the core says so" % path,
+ (key(ctx) != key(cache_ctx(Q_ORDER_B)))
+ == (_question_schema(Q_ORDER_A) != _question_schema(Q_ORDER_B)))
+ check_true("%s/the same call is a hit" % path, key(ctx) == key(cache_ctx(Q_ORDER_A)))
+ check_true("%s/another checkpoint is a new entry" % path,
+ key(ctx) != key(cache_ctx(Q_ORDER_A, model="multilingual")))
+ for field in ("max_len", "head_max_len"):
+ check_true("%s/another %s is a new entry" % (path, field),
+ key(ctx) != key(cache_ctx(Q_ORDER_A, **{field: 256})))
+
+# Three copies, one key: a docs page that drifts from the example fails here, not in someone's
+# production cache.
+for path, key in TAUGHT[1:]:
+ check("%s/keyed like the example" % path, key(cache_ctx(Q_ORDER_A)), TAUGHT[0][1](cache_ctx(Q_ORDER_A)))
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)