Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(examples): let the cache hook cover every state the call carries

Aashish8b332fe0c00c
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index 8817e666..50639995 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -66,9 +66,9 @@ pass is skipped; `on_predict_end` still runs and the supplied results are return
```python
def cache_read(ctx):
- hit = CACHE.get(key(ctx.states[0], ctx.questions))
- if hit is not None:
- ctx.skip([hit]) # list of per-state results, same shape as predict_batch's return
+ hits = [CACHE.get(key(s, ctx.questions)) for s in ctx.states]
+ if all(hit is not None for hit in hits):
+ ctx.skip(hits) # one entry per state in ctx.states, same shape as predict_batch's return
```
On the `Router`, a skipped payload gets a `routing` key added (without overwriting one it
diff --git a/docs/hooks/examples.md b/docs/hooks/examples.md
index 27e78437..66bb6852 100644
--- a/docs/hooks/examples.md
+++ b/docs/hooks/examples.md
@@ -95,21 +95,21 @@ import laya
CACHE = {}
-def key(ctx):
+def key(ctx, index):
# Not sort_keys=True: criteria order is positional, so two orders are two questions,
# and the checkpoint and token budget change the answer too.
- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,
ctx.max_len, ctx.head_max_len], default=str)
return hashlib.sha256(payload.encode()).hexdigest()
def read(ctx):
- hit = CACHE.get(key(ctx))
- if hit is not None:
- ctx.skip([hit])
+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]
+ if all(hit is not None for hit in hits):
+ ctx.skip(hits) # one per state: skip replaces the whole call
def write(ctx):
- if ctx.results:
- CACHE[key(ctx)] = ctx.results[0]
+ for i, result in enumerate(ctx.results or []):
+ CACHE[key(ctx, i)] = result
agent = laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)
first = agent.system_one("state", QUESTIONS) # runs the model
diff --git a/docs/hooks/patterns.md b/docs/hooks/patterns.md
index 198779ad..344d9d54 100644
--- a/docs/hooks/patterns.md
+++ b/docs/hooks/patterns.md
@@ -88,27 +88,29 @@ import hashlib, json
CACHE = {}
-def key(ctx):
+def key(ctx, index):
# Not sort_keys=True: criteria order is positional, so two orders are two questions,
# and the checkpoint and token budget change the answer too.
- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,
ctx.max_len, ctx.head_max_len], default=str)
return hashlib.sha256(payload.encode()).hexdigest()
def read(ctx):
- hit = CACHE.get(key(ctx))
- if hit is not None:
- ctx.skip([hit])
+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]
+ if all(hit is not None for hit in hits):
+ ctx.skip(hits) # one per state: skip replaces the whole call
def write(ctx):
- if ctx.results:
- CACHE[key(ctx)] = ctx.results[0]
+ for i, result in enumerate(ctx.results or []):
+ CACHE[key(ctx, i)] = result
laya.load("convaiinnovations/laya", on_predict_start=read, on_predict_end=write)
```
-Guard the cache with a lock when serving concurrently. On the Router the cached payload still
-gets a `routing` key, so the return shape is unchanged.
+Hooks fire once per call, so keying on one state is not enough on `predict_batch`: `ctx.skip()`
+replaces every result the call would have returned. Guard the cache with a lock when serving
+concurrently. On the Router the cached payload still gets a `routing` key, so the return shape is
+unchanged.
### Metrics
diff --git a/examples/hooks/cache.py b/examples/hooks/cache.py
index 4adea990..3aa1d83e 100644
--- a/examples/hooks/cache.py
+++ b/examples/hooks/cache.py
@@ -1,8 +1,9 @@
"""Cache decisions, and skip inference on a hit.
A start hook calls `ctx.skip(results)`; the engine skips the forward pass and still runs
-the end hooks. The key has to cover everything the answer depends on, so it takes the whole
-context rather than just the state and the questions.
+the end hooks. The key has to cover everything the answer depends on, and a hook has to cover
+everything the call carries: hooks fire once per call, and `predict_batch` calls them with
+every state of the batch at once.
python examples/hooks/cache.py
"""
@@ -14,8 +15,8 @@ import laya
CACHE = {}
-def cache_key(ctx):
- """One entry per question the model is actually asked.
+def cache_key(ctx, index):
+ """One entry per question the model is actually asked, for one state of the call.
Deliberately not `sort_keys=True`: a choice question's criteria order is positional, so two
orders are two questions, and `_question_schema` in `laya/router.py` keeps them apart for the
@@ -23,26 +24,27 @@ def cache_key(ctx):
caller's numbers. `ctx.model` and the token budget belong in the key for the same reason: on
the Router one hook set serves three checkpoints, and a smaller `max_len` truncates the state.
"""
- payload = json.dumps([ctx.states[0], ctx.questions, ctx.model,
+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,
ctx.max_len, ctx.head_max_len], default=str)
return hashlib.sha256(payload.encode()).hexdigest()
def cache_read(ctx):
- hit = CACHE.get(cache_key(ctx))
- if hit is not None:
- ctx.skip([hit])
+ hits = [CACHE.get(cache_key(ctx, i)) for i in range(len(ctx.states))]
+ if all(hit is not None for hit in hits):
+ ctx.skip(hits) # one per state: `skip` replaces the whole call, not its first result
def cache_write(ctx):
- if ctx.results:
- CACHE[cache_key(ctx)] = ctx.results[0]
+ for i, result in enumerate(ctx.results or []):
+ CACHE[cache_key(ctx, i)] = result
agent = laya.load("convaiinnovations/laya",
on_predict_start=cache_read, on_predict_end=cache_write)
STATE = "I was charged twice for the same invoice."
+OTHER = "Where do I change my notification settings?"
CRITERIA = {"refund": "give me money back", "cancel": "stop the service",
"other": "anything else"}
QUESTIONS = {"ask": {"type": "choice", "instructions": "What does the customer want?",
@@ -57,3 +59,9 @@ again = agent.system_one(STATE, QUESTIONS) # served from CACHE, no forward p
agent.system_one(STATE, REORDERED) # runs too: a reordered question is a new entry
print("cache entries:", len(CACHE)) # 2, not 1
print("same answer:", first["answers"] == again["answers"])
+
+# Batched, in the other order: both states are cached, so the whole call is served.
+served = agent.predict_batch([OTHER, STATE], QUESTIONS)
+print("results for a 2-state batch:", len(served))
+print("one answer per state, in the caller's order:",
+ [r["answers"]["ask"]["choice"] for r in served])
diff --git a/laya/hooks.py b/laya/hooks.py
index 58059b8d..b3afbd87 100644
--- a/laya/hooks.py
+++ b/laya/hooks.py
@@ -46,7 +46,12 @@ class PredictContext:
error: Optional[BaseException] = None
def skip(self, results: List[Dict[str, Any]]) -> None:
- """Set cached results from a start hook; inference is skipped, end hooks still run."""
+ """Set cached results from a start hook; inference is skipped, end hooks still run.
+
+ `results` replaces the whole call, so it carries one entry per state in `ctx.states` --
+ the shape `predict_batch` returns -- in that order. A hook fires once per call, and a
+ call can carry many states.
+ """
self.results = results
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index d4e1e640..a8dd0419 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -10,6 +10,7 @@ the key matched, so what a key covers is part of the contract, not an implementa
Run: python tests/test_hooks_api.py
"""
import dataclasses
+import functools
import hashlib
import inspect
import json
@@ -180,33 +181,57 @@ Q_ORDER_A = {"ask": {"type": "choice", "instructions": "What does the customer w
Q_ORDER_B = {"ask": {"type": "choice", "instructions": "What does the customer want?",
"criteria": {"other": CRITERIA["other"], "refund": CRITERIA["refund"],
"cancel": CRITERIA["cancel"]}}}
+BATCH = ["I was charged twice for the same invoice.",
+ "The app crashes every time I open the export screen.",
+ "Where do I change my notification settings?"]
-def taught_key(path, marker):
- """The key builder of one of those three files, exec'd out of that file's own text.
+def taught_cache(path):
+ """One taught copy of the cache pattern, exec'd out of that file's own text.
Sliced rather than imported: the example calls `laya.load()` at module scope, which would
- download a checkpoint, and a docs code block is not importable at all.
+ download a checkpoint, and a docs code block is not importable at all. The slice stops at that
+ `laya.load(...)` line for the same reason, so none of this needs weights.
"""
with open(os.path.join(REPO, path), encoding="utf-8") as handle:
text = handle.read()
- start = text.index("def %s(" % marker)
- namespace = {"json": json, "hashlib": hashlib}
- exec(text[start:text.index("\ndef ", start + 1)], namespace)
- return namespace[marker]
+ start = text.index("CACHE = {")
+ stop = len(text)
+ for cut in ("\nlaya.load", "\nagent = laya.load"):
+ found = text.find(cut, start)
+ if found != -1:
+ stop = min(stop, found)
+ namespace = {"json": json, "hashlib": hashlib, "laya": laya}
+ exec(text[start:stop], namespace)
+ return namespace
-def cache_ctx(questions, model="english", max_len=None, head_max_len=None):
- return PredictContext(states=["I was charged twice for the same invoice."],
- questions=questions, model=model, max_len=max_len,
- head_max_len=head_max_len)
+def key_of(namespace, ctx, index=0):
+ """The copy's key builder, called the way that copy declares it."""
+ key = namespace.get("cache_key") or namespace["key"]
+ return key(ctx, index) if len(sig(key)) > 1 else key(ctx)
-TAUGHT = [("examples/hooks/cache.py", taught_key("examples/hooks/cache.py", "cache_key")),
- ("docs/hooks/patterns.md", taught_key("docs/hooks/patterns.md", "key")),
- ("docs/hooks/examples.md", taught_key("docs/hooks/examples.md", "key"))]
+def cache_ctx(questions, model="english", max_len=None, head_max_len=None, states=(BATCH[0],)):
+ return PredictContext(states=list(states), questions=questions, model=model,
+ max_len=max_len, head_max_len=head_max_len)
-for path, key in TAUGHT:
+
+def answers_for(states):
+ """Per-state payloads, each labelled with its own state, so a mix-up is visible."""
+ return [{"model": "laya-rl-agent", "answers": {"ask": {"type": "choice", "choice": tag}},
+ "usage": {"input_tokens": 10 + i}} for i, tag in enumerate(
+ ["billing", "support", "other", "refund", "cancel"][:len(states)])]
+
+
+TAUGHT = [("examples/hooks/cache.py", taught_cache("examples/hooks/cache.py")),
+ ("docs/hooks/patterns.md", taught_cache("docs/hooks/patterns.md")),
+ ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md"))]
+
+for path, namespace in TAUGHT:
+ key = functools.partial(key_of, namespace)
+ read = namespace.get("cache_read") or namespace["read"]
+ write = namespace.get("cache_write") or namespace["write"]
ctx = cache_ctx(Q_ORDER_A)
check_true("%s/reordered criteria is a new entry" % path, key(ctx) != key(cache_ctx(Q_ORDER_B)))
check_true("%s/reorders are distinct exactly when the core says so" % path,
@@ -219,10 +244,45 @@ for path, key in TAUGHT:
check_true("%s/another %s is a new entry" % (path, field),
key(ctx) != key(cache_ctx(Q_ORDER_A, **{field: 256})))
+ # A hook fires once per call, and a call can carry a whole batch.
+ check("%s/key takes the state it describes" % path,
+ len(sig(namespace.get("cache_key") or namespace["key"])), 2)
+ multi = cache_ctx(Q_ORDER_A, states=BATCH)
+ check_true("%s/two states of one call are two entries" % path,
+ key(multi, 0) != key(multi, 1))
+ namespace["CACHE"].clear()
+ filled = cache_ctx(Q_ORDER_A, states=BATCH)
+ filled.results = answers_for(BATCH)
+ write(filled)
+ check("%s/write stores one entry per state" % path, len(namespace["CACHE"]), len(BATCH))
+ warm = cache_ctx(Q_ORDER_A, states=list(reversed(BATCH)))
+ read(warm)
+ check("%s/a warm batch is served back per state, in the caller's order" % path,
+ warm.results, list(reversed(filled.results)))
+ cold = cache_ctx(Q_ORDER_A, states=BATCH[:2] + ["an uncached state"])
+ read(cold)
+ check_true("%s/a batch with one uncached state still runs" % path, cold.results is None)
+ single = cache_ctx(Q_ORDER_A, states=[BATCH[1]])
+ read(single)
+ check("%s/a single-state call is one entry" % path, single.results, [filled.results[1]])
+ empty = cache_ctx(Q_ORDER_A, states=[])
+ try:
+ read(empty)
+ served = empty.results
+ except Exception as exc: # a hook that raises on an empty call is a failure
+ served = "raised %s" % exc.__class__.__name__
+ check("%s/an empty batch skips to an empty list" % path, served, [])
+ namespace["CACHE"].clear()
+
# Three copies, one key: a docs page that drifts from the example fails here, not in someone's
# production cache.
-for path, key in TAUGHT[1:]:
- check("%s/keyed like the example" % path, key(cache_ctx(Q_ORDER_A)), TAUGHT[0][1](cache_ctx(Q_ORDER_A)))
+for path, namespace in TAUGHT[1:]:
+ check("%s/keyed like the example" % path,
+ key_of(namespace, cache_ctx(Q_ORDER_A)),
+ key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A)))
+ check("%s/per-state like the example" % path,
+ key_of(namespace, cache_ctx(Q_ORDER_A, states=BATCH), 1),
+ key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A, states=BATCH), 1))
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))