Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(common): serialise fast-tokenizer encoding across threads
thread3d850563c2c231
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 11818be5..7ef48da3 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -103,6 +103,7 @@ jobs:python tests/test_packaging.pypython tests/test_doc_tables.pypython tests/test_tokenizer_cache.py+ python tests/test_tokenizer_concurrency.pypython tests/test_lazy_import.pypython tests/test_runtime_fixes.pypython tests/test_lang_guess.pydiff --git a/laya/agent.py b/laya/agent.pyindex a76041f7..c158e263 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -23,6 +23,7 @@ from .common import (answer_confidence,confidence_from_probs,_resolve_noul_labels,+ encode_text,render_options,serialize_state,temp_bucket,@@ -573,7 +574,8 @@ class Agent(HookRegistry):# re-serializing and re-tokenizing it inside build_sequence per question was pure# duplicated work. Tokenize in full and let build_sequence slice per question, so# left-truncation for conversation lists keeps its meaning.- state_ids = self.tok(+ state_ids = encode_text(+ self.tok,serialize_state(state).replace(self.tok.mask_token, " "),add_special_tokens=False,)["input_ids"]diff --git a/laya/common.py b/laya/common.pyindex 683b8a45..36960bbb 100644--- a/laya/common.py+++ b/laya/common.py@@ -2,6 +2,7 @@import jsonimport mathimport os+import threadingfrom typing import Dict, List, Optional, Unionimport numpy as np@@ -13,6 +14,20 @@ QTYPES = {"choice": 0, "score": 1, "noul": 2}QTYPE_NAMES = {v: k for k, v in QTYPES.items()}_DEFAULT_NOUL_LABELS = {"false": "false", "true": "true"}+# A fast tokenizer is not read-only: `truncation=True` / `padding=True` make it call+# `enable_truncation` / `enable_padding`, which mutates the shared Rust object. One tokenizer is+# parsed per checkpoint directory and shared by every Agent that wants it, so concurrent+# `predict()` calls -- on one Agent or on two sharing the cache -- raced and raised+# `RuntimeError: Already borrowed`. Serialise encoding instead: it is a small fraction of a call+# next to the forward pass, and this keeps the cache's single parse.+_TOKENIZE_LOCK = threading.RLock()+++def encode_text(tok, text, **kwargs):+ """Tokenize `text` while holding the lock a shared fast tokenizer needs."""+ with _TOKENIZE_LOCK:+ return tok(text, **kwargs)+def serialize_state(state: Union[str, dict, list]) -> str:if isinstance(state, str):@@ -94,13 +109,14 @@ def build_sequence(opts = render_options(q)order = option_order if option_order is not None else list(range(len(opts)))ins = str(q["ins"]).replace(mask_tok, " ")- head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]+ head_ids = encode_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]opt_ids = []for i in order:# Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the# whole (possibly long) description. truncation=True, max_length=48 keeps the first 48# tokens, which is exactly what the previous slice produced.- opt_tokens = tok(+ opt_tokens = encode_text(+ tok," " + opts[i].replace(mask_tok, " "),add_special_tokens=False,truncation=True,@@ -121,7 +137,8 @@ def build_sequence(ids.append(tok.sep_token_id)room = max(0, max_len - len(ids) - 1)if state_ids is None:- state_ids = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]+ state_ids = encode_text(tok, serialize_state(state).replace(mask_tok, " "),+ add_special_tokens=False)["input_ids"]# not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of itst = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]ids = ids + st + [tok.sep_token_id]diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 52326404..cb0d9e5d 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -14,6 +14,7 @@ from laya.common import (build_sequence,collate_items,confidence_from_probs,+ encode_text,render_options,serialize_state,temp_bucket,@@ -239,7 +240,8 @@ class ONNXAgent(HookRegistry):# re-serializing and re-tokenizing the same document inside build_sequence per# question (the PyTorch Agent already does this via `state_ids`).truncate_left = isinstance(state, list)- state_ids = self.tok(+ state_ids = encode_text(+ self.tok,serialize_state(state).replace(self.tok.mask_token, " "),add_special_tokens=False,)["input_ids"]diff --git a/laya/shortlist.py b/laya/shortlist.pyindex d4606be5..f3b351cc 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -19,7 +19,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequenceimport numpy as np-from .common import render_options, serialize_state+from .common import encode_text, render_options, serialize_stateDEFAULT_SHORTLIST_K = 20@@ -143,7 +143,8 @@ def embed_fn_from_agent(parts: List[np.ndarray] = []for start in range(0, len(rows), batch_size):chunk = rows[start : start + batch_size]- encoded = tok(+ encoded = encode_text(+ tok,chunk,padding=True,truncation=True,diff --git a/tests/test_tokenizer_concurrency.py b/tests/test_tokenizer_concurrency.pynew file mode 100644index 00000000..24648e30--- /dev/null+++ b/tests/test_tokenizer_concurrency.py@@ -0,0 +1,82 @@+"""Concurrent sequence building must not race on the shared fast tokenizer.++`build_sequence` asks the tokenizer for truncated option text (`truncation=True`), and that mutates+the underlying Rust object through `enable_truncation`. One tokenizer is parsed per checkpoint+directory and shared across Agents, so two threads encoding at once used to raise+`RuntimeError: Already borrowed`. `common.encode_text` serialises those calls; this guards it.+"""+import os+import sys+import threading++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from tokenizers import Tokenizer # noqa: E402+from tokenizers.models import WordLevel # noqa: E402+from transformers import PreTrainedTokenizerFast # noqa: E402++from laya.common import build_sequence # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s%s" % (name, (": " + detail) if detail else ""))+++VOCAB = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}+for i in range(200):+ VOCAB["tok%d" % i] = 5 + i++TOKENIZER = PreTrainedTokenizerFast(+ tokenizer_object=Tokenizer(WordLevel(VOCAB, unk_token="[UNK]")),+ pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",+ sep_token="[SEP]", mask_token="[MASK]",+)++# Long enough that truncation does real work on every option, not just enable itself.+QUESTION = {+ "t": "choice",+ "ins": "Which team should handle `message`?",+ "crit": {"billing": " ".join("tok%d" % (i * 3 % 200) for i in range(80)),+ "technical": " ".join("tok%d" % (i * 7 % 200) for i in range(80))},+}+STATE = " ".join("tok%d" % (i * 11 % 200) for i in range(600))++errors, results = [], []+++def worker():+ try:+ for _ in range(40):+ seq, markers = build_sequence(TOKENIZER, STATE, QUESTION, 512, 192)+ results.append((tuple(seq), tuple(markers)))+ except Exception as e: # noqa: BLE001+ errors.append("%s: %s" % (type(e).__name__, e))+++threads = [threading.Thread(target=worker) for _ in range(4)]+for t in threads:+ t.start()+for t in threads:+ t.join()++check_true("concurrency/four threads finish without an error", not errors,+ errors[0] if errors else "")+check("concurrency/every thread built the same sequence", len(set(results)), 1)+check("concurrency/all calls produced output", len(results), 4 * 40)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+sys.exit(1 if FAIL else 0)