Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(agent): check that predict_long encodes under the tokenizer lock

abhijithneilabraham3c63f08271c7
diff --git a/tests/test_tokenizer_concurrency.py b/tests/test_tokenizer_concurrency.py
index 24648e30..c0354344 100644
--- a/tests/test_tokenizer_concurrency.py
+++ b/tests/test_tokenizer_concurrency.py
@@ -15,7 +15,8 @@ from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from transformers import PreTrainedTokenizerFast # noqa: E402
-from laya.common import build_sequence # noqa: E402
+from laya.agent import Agent # noqa: E402
+from laya.common import _TOKENIZE_LOCK, build_sequence # noqa: E402
PASS, FAIL = [], []
@@ -76,6 +77,58 @@ check_true("concurrency/four threads finish without an error", not errors,
check("concurrency/every thread built the same sequence", len(set(results)), 1)
check("concurrency/all calls produced output", len(results), 4 * 40)
+
+# `Agent.predict_long` tokenizes the whole state before it splits it into windows. That encode has
+# to hold the same lock: a scan running next to a `predict()` on the shared tokenizer otherwise
+# raises `RuntimeError: Already borrowed` in both calls.
+class _LockProbeTok:
+ """A tokenizer stand-in that records every encode made while the tokenize lock was free."""
+
+ mask_token = "[MASK]"
+ cls_token_id, sep_token_id, mask_token_id = 2, 3, 4
+
+ def __init__(self):
+ self.calls, self.unlocked = 0, 0
+
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ free = []
+
+ def probe():
+ # The lock is re-entrant, so only another thread can tell whether it is held.
+ if _TOKENIZE_LOCK.acquire(blocking=False):
+ _TOKENIZE_LOCK.release()
+ free.append(True)
+
+ t = threading.Thread(target=probe)
+ t.start()
+ t.join()
+ self.calls += 1
+ self.unlocked += len(free)
+ ids = list(range(len(text)))
+ return {"input_ids": ids[:max_length] if (truncation and max_length) else ids}
+
+ def decode(self, ids):
+ return "w%d_%d" % (ids[0], ids[-1]) if ids else "w"
+
+
+def _scan_agent():
+ a = Agent.__new__(Agent)
+ a.cfg = {"max_len": 100, "head_max_len": 20}
+ a.tok = _LockProbeTok()
+ a._to_internal = staticmethod(Agent._to_internal).__func__
+ answer = {"flag": {"type": "noul", "noul": 0.5, "confidence": 0.5, "answer_confidence": 0.5}}
+ a.predict_batch = lambda states, questions, **kw: [
+ {"model": "stub", "answers": dict(answer), "usage": {"input_tokens": 1}} for _ in states]
+ return a
+
+
+scan_agent = _scan_agent()
+scanned = scan_agent.predict_long("x" * 400, {"flag": {"type": "noul", "instructions": "?"}})
+check_true("predict_long/the document was split into several windows", scanned["usage"]["windows"] > 1,
+ "windows=%r" % scanned["usage"]["windows"])
+check_true("predict_long/the tokenizer was called", scan_agent.tok.calls > 0)
+check("predict_long/every encode holds the tokenize lock", scan_agent.tok.unlocked, 0)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)