Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): don't rewrite tokenizer_config through the HuggingFace cache symlink (#107)
Emiliano G.O.8e631364d5ac
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ecd12925..aebbab68 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -64,6 +64,7 @@ jobs:python tests/test_decision_model.pypython tests/test_head_checkpointing.pypython tests/test_packaging.py+ python tests/test_tokenizer_cache.pypython tests/test_lang_guess.pypython tests/test_calibration_persistence.pypython tests/test_context_manager.pydiff --git a/laya/agent.py b/laya/agent.pyindex 7538ef34..9a9f59c4 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -46,10 +46,20 @@ def _fix_tokenizer_config(path: str):tcfg["extra_special_tokens"] = {"extra_%d" % i: t for i, t in enumerate(extra)}changed = Trueif changed:+ # HuggingFace snapshots are symlinks into a shared blob store. Writing through the+ # link would truncate a file shared with other revisions and processes, race+ # concurrent loads, and desync the hub's cache metadata. Detach the local file first.+ if os.path.islink(cfg_file):+ os.unlink(cfg_file)with open(cfg_file, "w") as f:json.dump(tcfg, f, indent=2)- except Exception:- pass+ except Exception as e:+ # Do not swallow this silently: if the patch did not apply, AutoTokenizer may fail later+ # with a confusing error and no hint that the config was the cause.+ warnings.warn(+ "laya: could not patch %s (%s); the tokenizer may fail to load with this "+ "transformers version." % (cfg_file, e),+ RuntimeWarning, stacklevel=2)def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, torch.Tensor], model_id: str):diff --git a/tests/test_tokenizer_cache.py b/tests/test_tokenizer_cache.pynew file mode 100644index 00000000..2c7f5704--- /dev/null+++ b/tests/test_tokenizer_cache.py@@ -0,0 +1,77 @@+"""`_fix_tokenizer_config` must not write through the HuggingFace snapshot symlink.++`snapshot_download` lays snapshots out as symlinks into a shared `blobs/` store, so opening+the snapshot file for writing truncates the shared blob. This test builds that exact layout+and asserts the blob survives.+"""+import os+import shutil+import sys+import tempfile+import warnings++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import _fix_tokenizer_config # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++root = tempfile.mkdtemp(prefix="laya_cache_")+blob_dir = os.path.join(root, "models--x", "blobs")+snap_dir = os.path.join(root, "models--x", "snapshots", "rev1", "tokenizer")+os.makedirs(blob_dir)+os.makedirs(snap_dir)++ORIGINAL = '{"tokenizer_class": "TokenizersBackend", "backend": "x", "is_local": true}'+blob = os.path.join(blob_dir, "deadbeef")+with open(blob, "w") as f:+ f.write(ORIGINAL)++link = os.path.join(snap_dir, "tokenizer_config.json")+os.symlink(blob, link)++check("setup/snapshot file is a symlink", os.path.islink(link), True)++_fix_tokenizer_config(os.path.dirname(snap_dir))++check("blob/is untouched", open(blob).read(), ORIGINAL)+check("snapshot/is now a regular file", os.path.islink(link), False)+check("snapshot/carries the patch", '"PreTrainedTokenizerFast"' in open(link).read(), True)+check("snapshot/drops backend+is_local as intended", '"backend": "x"' not in open(link).read(), True)++# a second call is a no-op and must not disturb anything either+_fix_tokenizer_config(os.path.dirname(snap_dir))+check("idempotent/blob still untouched", open(blob).read(), ORIGINAL)++# an unparseable config must warn (not crash, not silently do nothing)+bad_snap = os.path.join(root, "bad", "tokenizer")+os.makedirs(bad_snap)+bad_file = os.path.join(bad_snap, "tokenizer_config.json")+with open(bad_file, "w") as f:+ f.write("{ not json")+with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ _fix_tokenizer_config(os.path.dirname(bad_snap))+check(+ "bad-json/warns instead of raising",+ any(issubclass(w.category, RuntimeWarning) for w in caught),+ True,+)+check("bad-json/file left as written", open(bad_file).read(), "{ not json")++shutil.rmtree(root, ignore_errors=True)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+if not FAIL:+ print("all tokenizer-cache tests passed")+sys.exit(1 if FAIL else 0)