Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(hooks): make api.md teach the key the example uses

Aashishfbe27d124308
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index 50639995..ef32f475 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -64,13 +64,35 @@ class PredictContext:
Short-circuits inference. Called from `on_predict_start`, it sets `ctx.results` so the forward
pass is skipped; `on_predict_end` still runs and the supplied results are returned.
+The key has to cover everything the answer depends on, and the hook has to cover everything the
+call carries: hooks fire once per call, and `predict_batch` calls them with every state at once.
+
```python
+CACHE = {}
+
+def key(ctx, index):
+ # Not sort_keys=True: criteria order is positional, so two orders are two questions,
+ # and the checkpoint and token budget change the answer too.
+ payload = json.dumps([ctx.states[index], ctx.questions, ctx.model,
+ ctx.max_len, ctx.head_max_len], default=str)
+ return hashlib.sha256(payload.encode()).hexdigest()
+
def cache_read(ctx):
- hits = [CACHE.get(key(s, ctx.questions)) for s in ctx.states]
+ hits = [CACHE.get(key(ctx, i)) for i in range(len(ctx.states))]
if all(hit is not None for hit in hits):
- ctx.skip(hits) # one entry per state in ctx.states, same shape as predict_batch's return
+ ctx.skip(hits) # one entry per state, same shape as predict_batch's return
+
+def cache_write(ctx):
+ for i, result in enumerate(ctx.results or []):
+ CACHE[key(ctx, i)] = result
+
+laya.load("convaiinnovations/laya", on_predict_start=cache_read, on_predict_end=cache_write)
```
+`tests/test_hooks_api.py` execs this block, `examples/hooks/cache.py`, and the caching blocks of
+`docs/hooks/patterns.md` and `docs/hooks/examples.md`, and asserts all four keys the same way, so a
+page cannot teach a key the example has moved on from.
+
On the `Router`, a skipped payload gets a `routing` key added (without overwriting one it
already has), so `Router.predict` keeps its documented return shape.
diff --git a/examples/hooks/cache.py b/examples/hooks/cache.py
index 3aa1d83e..147cabc0 100644
--- a/examples/hooks/cache.py
+++ b/examples/hooks/cache.py
@@ -13,6 +13,7 @@ import json
import laya
CACHE = {}
+SKIPS = [] # one entry per call `cache_read` served, so the prints below are checkable
def cache_key(ctx, index):
@@ -32,6 +33,7 @@ def cache_key(ctx, index):
def cache_read(ctx):
hits = [CACHE.get(cache_key(ctx, i)) for i in range(len(ctx.states))]
if all(hit is not None for hit in hits):
+ SKIPS.append(len(hits))
ctx.skip(hits) # one per state: `skip` replaces the whole call, not its first result
@@ -60,8 +62,12 @@ agent.system_one(STATE, REORDERED) # runs too: a reordered question
print("cache entries:", len(CACHE)) # 2, not 1
print("same answer:", first["answers"] == again["answers"])
+# Warm the second state before the batch. `cache_read` skips only when *every* state of the call
+# has an entry, so a batch that mixes one warm state with one cold state runs a full forward.
+agent.system_one(OTHER, QUESTIONS)
# Batched, in the other order: both states are cached, so the whole call is served.
served = agent.predict_batch([OTHER, STATE], QUESTIONS)
print("results for a 2-state batch:", len(served))
print("one answer per state, in the caller's order:",
[r["answers"]["ask"]["choice"] for r in served])
+print("calls served without a forward pass:", len(SKIPS)) # 2: `again`, and this batch
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index a8dd0419..bf457866 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -9,10 +9,12 @@ the key matched, so what a key covers is part of the contract, not an implementa
Run: python tests/test_hooks_api.py
"""
+import contextlib
import dataclasses
import functools
import hashlib
import inspect
+import io
import json
import os
import sys
@@ -166,8 +168,9 @@ for label, cls in (("Agent", Agent), ("Router", Router), ("ONNXAgent", ONNXAgent
# ------------------------------------------------- the cache key the examples and docs teach
-# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md` and
-# `docs/hooks/examples.md`, are three copies of one `ctx.skip()` pattern. The key is the part that
+# `examples/hooks/cache.py`, and the caching blocks of `docs/hooks/patterns.md`,
+# `docs/hooks/examples.md` and `docs/hooks/api.md`, are four copies of one `ctx.skip()` pattern.
+# The key is the part that
# can be wrong: a payload sorted by key folds two criteria orders into one cache entry, while
# `_question_schema` keeps them apart because a choice question's option order is positional. The
# second caller then gets the first caller's answer. Measured on the English checkpoint, a question
@@ -191,13 +194,14 @@ def taught_cache(path):
Sliced rather than imported: the example calls `laya.load()` at module scope, which would
download a checkpoint, and a docs code block is not importable at all. The slice stops at that
- `laya.load(...)` line for the same reason, so none of this needs weights.
+ `laya.load(...)` line, or at the closing fence of the block, for the same reason, so none of
+ this needs weights.
"""
with open(os.path.join(REPO, path), encoding="utf-8") as handle:
text = handle.read()
start = text.index("CACHE = {")
stop = len(text)
- for cut in ("\nlaya.load", "\nagent = laya.load"):
+ for cut in ("\nlaya.load", "\nagent = laya.load", "\n```"):
found = text.find(cut, start)
if found != -1:
stop = min(stop, found)
@@ -226,7 +230,10 @@ def answers_for(states):
TAUGHT = [("examples/hooks/cache.py", taught_cache("examples/hooks/cache.py")),
("docs/hooks/patterns.md", taught_cache("docs/hooks/patterns.md")),
- ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md"))]
+ ("docs/hooks/examples.md", taught_cache("docs/hooks/examples.md")),
+ # `api.md` is the page that documents `ctx.skip()`, so it is the page a reader copies
+ # from -- and it carries the same key, which makes it the copy most able to go stale.
+ ("docs/hooks/api.md", taught_cache("docs/hooks/api.md"))]
for path, namespace in TAUGHT:
key = functools.partial(key_of, namespace)
@@ -285,6 +292,67 @@ for path, namespace in TAUGHT[1:]:
key_of(TAUGHT[0][1], cache_ctx(Q_ORDER_A, states=BATCH), 1))
+# ------------------------------------------- what the cache example's own demo claims
+# The example closes by saying its two-state batch "is served", and nothing in its printed output
+# distinguishes a served payload from a fresh one -- both are the same answers. So run the demo
+# against a stub agent that counts its own forward passes, and check the claim rather than trusting
+# the comment. No weights: `laya.load` is the only thing the example takes from the library.
+def run_cache_example():
+ with open(os.path.join(REPO, "examples/hooks/cache.py"), encoding="utf-8") as handle:
+ source = handle.read().replace("\nimport laya\n", "\n")
+ forwards = []
+
+ class StubAgent:
+ def __init__(self, on_start, on_end):
+ self.on_start, self.on_end = on_start, on_end
+
+ def _call(self, states, questions):
+ ctx = PredictContext(states=list(states), questions=questions, model="english")
+ if self.on_start:
+ self.on_start(ctx)
+ if ctx.results is None:
+ forwards.append(len(states))
+ ctx.results = [{"model": "laya-rl-agent",
+ "answers": {"ask": {"type": "choice", "choice": s[:8]}}}
+ for s in states]
+ ctx.usage = {"input_tokens": 10 * len(states), "output_tokens": 0}
+ if self.on_end:
+ self.on_end(ctx)
+ return ctx.results
+
+ def system_one(self, state, questions, model=None):
+ return self._call([state], questions)[0]
+
+ def predict_batch(self, states, questions, model=None, batch_size=None):
+ return self._call(states, questions)
+
+ class StubLaya:
+ @staticmethod
+ def load(name, on_predict_start=None, on_predict_end=None):
+ return StubAgent(on_predict_start, on_predict_end)
+
+ out = io.StringIO()
+ namespace = {"laya": StubLaya}
+ with contextlib.redirect_stdout(out):
+ exec(compile(source, "examples/hooks/cache.py", "exec"), namespace)
+ return namespace, forwards, out.getvalue()
+
+
+example, example_forwards, example_out = run_cache_example()
+# Why the exact list: the repeat of STATE and the warm batch are the two served calls, and the
+# batch is the only multi-state one. A demo that mixed a cold state into the batch would show
+# `[1]` here, which is the claim its comment does not make.
+check("examples/hooks/cache.py/the repeat and the batch are the served calls",
+ example["SKIPS"], [1, 2])
+check("examples/hooks/cache.py/one forward per new entry, none for the batch",
+ example_forwards, [1, 1, 1])
+check_true("examples/hooks/cache.py/its own output still says two entries",
+ "cache entries: 2" in example_out, example_out)
+check("examples/hooks/cache.py/the batch answers in the caller's order",
+ [r["answers"]["ask"]["choice"] for r in example["served"]],
+ [example["OTHER"][:8], example["STATE"][:8]])
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)