Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(common): serialise fast-tokenizer encoding across threads

thread3d850563c2c231
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 11818be5..7ef48da3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -103,6 +103,7 @@ jobs:
python tests/test_packaging.py
python tests/test_doc_tables.py
python tests/test_tokenizer_cache.py
+ python tests/test_tokenizer_concurrency.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
python tests/test_lang_guess.py
diff --git a/laya/agent.py b/laya/agent.py
index a76041f7..c158e263 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -23,6 +23,7 @@ from .common import (
answer_confidence,
confidence_from_probs,
_resolve_noul_labels,
+ encode_text,
render_options,
serialize_state,
temp_bucket,
@@ -573,7 +574,8 @@ class Agent(HookRegistry):
# re-serializing and re-tokenizing it inside build_sequence per question was pure
# duplicated work. Tokenize in full and let build_sequence slice per question, so
# left-truncation for conversation lists keeps its meaning.
- state_ids = self.tok(
+ state_ids = encode_text(
+ self.tok,
serialize_state(state).replace(self.tok.mask_token, " "),
add_special_tokens=False,
)["input_ids"]
diff --git a/laya/common.py b/laya/common.py
index 683b8a45..36960bbb 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -2,6 +2,7 @@
import json
import math
import os
+import threading
from typing import Dict, List, Optional, Union
import numpy as np
@@ -13,6 +14,20 @@ QTYPES = {"choice": 0, "score": 1, "noul": 2}
QTYPE_NAMES = {v: k for k, v in QTYPES.items()}
_DEFAULT_NOUL_LABELS = {"false": "false", "true": "true"}
+# A fast tokenizer is not read-only: `truncation=True` / `padding=True` make it call
+# `enable_truncation` / `enable_padding`, which mutates the shared Rust object. One tokenizer is
+# parsed per checkpoint directory and shared by every Agent that wants it, so concurrent
+# `predict()` calls -- on one Agent or on two sharing the cache -- raced and raised
+# `RuntimeError: Already borrowed`. Serialise encoding instead: it is a small fraction of a call
+# next to the forward pass, and this keeps the cache's single parse.
+_TOKENIZE_LOCK = threading.RLock()
+
+
+def encode_text(tok, text, **kwargs):
+ """Tokenize `text` while holding the lock a shared fast tokenizer needs."""
+ with _TOKENIZE_LOCK:
+ return tok(text, **kwargs)
+
def serialize_state(state: Union[str, dict, list]) -> str:
if isinstance(state, str):
@@ -94,13 +109,14 @@ def build_sequence(
opts = render_options(q)
order = option_order if option_order is not None else list(range(len(opts)))
ins = str(q["ins"]).replace(mask_tok, " ")
- head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]
+ head_ids = encode_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]
opt_ids = []
for i in order:
# Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the
# whole (possibly long) description. truncation=True, max_length=48 keeps the first 48
# tokens, which is exactly what the previous slice produced.
- opt_tokens = tok(
+ opt_tokens = encode_text(
+ tok,
" " + opts[i].replace(mask_tok, " "),
add_special_tokens=False,
truncation=True,
@@ -121,7 +137,8 @@ def build_sequence(
ids.append(tok.sep_token_id)
room = max(0, max_len - len(ids) - 1)
if state_ids is None:
- state_ids = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
+ state_ids = encode_text(tok, serialize_state(state).replace(mask_tok, " "),
+ add_special_tokens=False)["input_ids"]
# not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of it
st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]
ids = ids + st + [tok.sep_token_id]
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 52326404..cb0d9e5d 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -14,6 +14,7 @@ from laya.common import (
build_sequence,
collate_items,
confidence_from_probs,
+ encode_text,
render_options,
serialize_state,
temp_bucket,
@@ -239,7 +240,8 @@ class ONNXAgent(HookRegistry):
# re-serializing and re-tokenizing the same document inside build_sequence per
# question (the PyTorch Agent already does this via `state_ids`).
truncate_left = isinstance(state, list)
- state_ids = self.tok(
+ state_ids = encode_text(
+ self.tok,
serialize_state(state).replace(self.tok.mask_token, " "),
add_special_tokens=False,
)["input_ids"]
diff --git a/laya/shortlist.py b/laya/shortlist.py
index d4606be5..f3b351cc 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -19,7 +19,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence
import numpy as np
-from .common import render_options, serialize_state
+from .common import encode_text, render_options, serialize_state
DEFAULT_SHORTLIST_K = 20
@@ -143,7 +143,8 @@ def embed_fn_from_agent(
parts: List[np.ndarray] = []
for start in range(0, len(rows), batch_size):
chunk = rows[start : start + batch_size]
- encoded = tok(
+ encoded = encode_text(
+ tok,
chunk,
padding=True,
truncation=True,
diff --git a/tests/test_tokenizer_concurrency.py b/tests/test_tokenizer_concurrency.py
new file mode 100644
index 00000000..24648e30
--- /dev/null
+++ b/tests/test_tokenizer_concurrency.py
@@ -0,0 +1,82 @@
+"""Concurrent sequence building must not race on the shared fast tokenizer.
+
+`build_sequence` asks the tokenizer for truncated option text (`truncation=True`), and that mutates
+the underlying Rust object through `enable_truncation`. One tokenizer is parsed per checkpoint
+directory and shared across Agents, so two threads encoding at once used to raise
+`RuntimeError: Already borrowed`. `common.encode_text` serialises those calls; this guards it.
+"""
+import os
+import sys
+import threading
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from tokenizers import Tokenizer # noqa: E402
+from tokenizers.models import WordLevel # noqa: E402
+from transformers import PreTrainedTokenizerFast # noqa: E402
+
+from laya.common import build_sequence # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s%s" % (name, (": " + detail) if detail else ""))
+
+
+VOCAB = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}
+for i in range(200):
+ VOCAB["tok%d" % i] = 5 + i
+
+TOKENIZER = PreTrainedTokenizerFast(
+ tokenizer_object=Tokenizer(WordLevel(VOCAB, unk_token="[UNK]")),
+ pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",
+ sep_token="[SEP]", mask_token="[MASK]",
+)
+
+# Long enough that truncation does real work on every option, not just enable itself.
+QUESTION = {
+ "t": "choice",
+ "ins": "Which team should handle `message`?",
+ "crit": {"billing": " ".join("tok%d" % (i * 3 % 200) for i in range(80)),
+ "technical": " ".join("tok%d" % (i * 7 % 200) for i in range(80))},
+}
+STATE = " ".join("tok%d" % (i * 11 % 200) for i in range(600))
+
+errors, results = [], []
+
+
+def worker():
+ try:
+ for _ in range(40):
+ seq, markers = build_sequence(TOKENIZER, STATE, QUESTION, 512, 192)
+ results.append((tuple(seq), tuple(markers)))
+ except Exception as e: # noqa: BLE001
+ errors.append("%s: %s" % (type(e).__name__, e))
+
+
+threads = [threading.Thread(target=worker) for _ in range(4)]
+for t in threads:
+ t.start()
+for t in threads:
+ t.join()
+
+check_true("concurrency/four threads finish without an error", not errors,
+ errors[0] if errors else "")
+check("concurrency/every thread built the same sequence", len(set(results)), 1)
+check("concurrency/all calls produced output", len(results), 4 * 40)
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+sys.exit(1 if FAIL else 0)