Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): don't rewrite tokenizer_config through the HuggingFace cache symlink (#107)

Emiliano G.O.8e631364d5ac
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ecd12925..aebbab68 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,6 +64,7 @@ jobs:
python tests/test_decision_model.py
python tests/test_head_checkpointing.py
python tests/test_packaging.py
+ python tests/test_tokenizer_cache.py
python tests/test_lang_guess.py
python tests/test_calibration_persistence.py
python tests/test_context_manager.py
diff --git a/laya/agent.py b/laya/agent.py
index 7538ef34..9a9f59c4 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -46,10 +46,20 @@ def _fix_tokenizer_config(path: str):
tcfg["extra_special_tokens"] = {"extra_%d" % i: t for i, t in enumerate(extra)}
changed = True
if changed:
+ # HuggingFace snapshots are symlinks into a shared blob store. Writing through the
+ # link would truncate a file shared with other revisions and processes, race
+ # concurrent loads, and desync the hub's cache metadata. Detach the local file first.
+ if os.path.islink(cfg_file):
+ os.unlink(cfg_file)
with open(cfg_file, "w") as f:
json.dump(tcfg, f, indent=2)
- except Exception:
- pass
+ except Exception as e:
+ # Do not swallow this silently: if the patch did not apply, AutoTokenizer may fail later
+ # with a confusing error and no hint that the config was the cause.
+ warnings.warn(
+ "laya: could not patch %s (%s); the tokenizer may fail to load with this "
+ "transformers version." % (cfg_file, e),
+ RuntimeWarning, stacklevel=2)
def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, torch.Tensor], model_id: str):
diff --git a/tests/test_tokenizer_cache.py b/tests/test_tokenizer_cache.py
new file mode 100644
index 00000000..2c7f5704
--- /dev/null
+++ b/tests/test_tokenizer_cache.py
@@ -0,0 +1,77 @@
+"""`_fix_tokenizer_config` must not write through the HuggingFace snapshot symlink.
+
+`snapshot_download` lays snapshots out as symlinks into a shared `blobs/` store, so opening
+the snapshot file for writing truncates the shared blob. This test builds that exact layout
+and asserts the blob survives.
+"""
+import os
+import shutil
+import sys
+import tempfile
+import warnings
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import _fix_tokenizer_config # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+root = tempfile.mkdtemp(prefix="laya_cache_")
+blob_dir = os.path.join(root, "models--x", "blobs")
+snap_dir = os.path.join(root, "models--x", "snapshots", "rev1", "tokenizer")
+os.makedirs(blob_dir)
+os.makedirs(snap_dir)
+
+ORIGINAL = '{"tokenizer_class": "TokenizersBackend", "backend": "x", "is_local": true}'
+blob = os.path.join(blob_dir, "deadbeef")
+with open(blob, "w") as f:
+ f.write(ORIGINAL)
+
+link = os.path.join(snap_dir, "tokenizer_config.json")
+os.symlink(blob, link)
+
+check("setup/snapshot file is a symlink", os.path.islink(link), True)
+
+_fix_tokenizer_config(os.path.dirname(snap_dir))
+
+check("blob/is untouched", open(blob).read(), ORIGINAL)
+check("snapshot/is now a regular file", os.path.islink(link), False)
+check("snapshot/carries the patch", '"PreTrainedTokenizerFast"' in open(link).read(), True)
+check("snapshot/drops backend+is_local as intended", '"backend": "x"' not in open(link).read(), True)
+
+# a second call is a no-op and must not disturb anything either
+_fix_tokenizer_config(os.path.dirname(snap_dir))
+check("idempotent/blob still untouched", open(blob).read(), ORIGINAL)
+
+# an unparseable config must warn (not crash, not silently do nothing)
+bad_snap = os.path.join(root, "bad", "tokenizer")
+os.makedirs(bad_snap)
+bad_file = os.path.join(bad_snap, "tokenizer_config.json")
+with open(bad_file, "w") as f:
+ f.write("{ not json")
+with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ _fix_tokenizer_config(os.path.dirname(bad_snap))
+check(
+ "bad-json/warns instead of raising",
+ any(issubclass(w.category, RuntimeWarning) for w in caught),
+ True,
+)
+check("bad-json/file left as written", open(bad_file).read(), "{ not json")
+
+shutil.rmtree(root, ignore_errors=True)
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+if not FAIL:
+ print("all tokenizer-cache tests passed")
+sys.exit(1 if FAIL else 0)