Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(verify): checkpoints.py repairs a truncated download instead of trusting it

Aashish4df855ef01c6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 39843b20..e03820d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -115,6 +115,7 @@ jobs:
python tests/test_onnx_long.py
python tests/test_onnx_sort.py
python tests/test_download.py
+ python tests/test_verify_checkpoints.py
python tests/test_shortlist.py
python tests/test_decision_model.py
python tests/test_head_checkpointing.py
@@ -300,7 +301,7 @@ jobs:
run: |
for t in test_router test_criteria test_batch test_predict_long \
test_structured test_structured_api test_onnx_lang_parity \
- test_download test_shortlist test_decision_model \
+ test_download test_verify_checkpoints test_shortlist test_decision_model \
test_head_checkpointing test_packaging test_doc_tables \
test_tokenizer_cache test_tokenizer_concurrency test_lazy_import \
test_runtime_fixes test_lang_guess test_identifier_complexity \
diff --git a/tests/test_verify_checkpoints.py b/tests/test_verify_checkpoints.py
new file mode 100644
index 00000000..514722bc
--- /dev/null
+++ b/tests/test_verify_checkpoints.py
@@ -0,0 +1,144 @@
+"""`verify/checkpoints.py`: what `--fetch` repairs, and which remedy the failure banner offers.
+
+The script promises the manifest catches "a truncated download or an upstream change ...
+rather than turning into quietly wrong answers". Two defects stood between a failure and its
+fix, both about the advice a user follows after one:
+
+1. `--fetch` skipped any checkpoint whose file existed, so the banner's instruction -- "fetch
+ it with `--fetch`" -- could never repair a truncated transfer. It now re-downloads when
+ the size disagrees with the manifest, and leaves a right-sized file alone (no network call).
+2. The banner offered one remedy for both causes: `--record`. For a truncated file that
+ writes the truncation into the manifest as the new truth. It now separates the two: a size
+ mismatch is re-fetched, an equal-sized digest change is the one to review and re-record.
+
+No network and no real weights: the manifest is rewritten into a temp directory and
+`huggingface_hub.snapshot_download` is a stub that writes the bytes the manifest expects.
+
+Run: python tests/test_verify_checkpoints.py
+"""
+import contextlib
+import hashlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+from unittest import mock
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "verify" / "checkpoints.py"
+
+spec = importlib.util.spec_from_file_location("verify_checkpoints_under_test", SCRIPT)
+check = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(check)
+
+GOOD = b"the checkpoint bytes, as fetched at the pinned revision"
+
+
+def manifest_for(relpath, name="english", good=GOOD):
+ return {
+ "repo": "convaiinnovations/laya",
+ "revision": "0" * 40,
+ "note": "fixture",
+ "checkpoints": [{
+ "name": name, "repo": "convaiinnovations/laya", "subfolder": None,
+ "path": relpath, "file": "model.safetensors",
+ "bytes": len(good), "sha256": hashlib.sha256(good).hexdigest(),
+ "params": 1, "encoder": "fixture", "context": 8,
+ }],
+ }
+
+
+class FakeHub:
+ """`huggingface_hub` stand-in: remembers the calls and writes the expected bytes."""
+
+ def __init__(self, payload=GOOD):
+ self.payload = payload
+ self.calls = []
+
+ def snapshot_download(self, repo, revision=None, local_dir=None, allow_patterns=None):
+ self.calls.append({"repo": repo, "revision": revision})
+ os.makedirs(local_dir, exist_ok=True)
+ with open(os.path.join(local_dir, "model.safetensors"), "wb") as handle:
+ handle.write(self.payload)
+ return local_dir
+
+
+class CheckpointsTests(unittest.TestCase):
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self._tmp.cleanup)
+ self.root = Path(self._tmp.name)
+ self.models = self.root / "models"
+ (self.models / "laya").mkdir(parents=True)
+
+ def write_manifest(self, **kwargs):
+ path = self.root / "checkpoints.json"
+ path.write_text(json.dumps(manifest_for("laya/model.safetensors", **kwargs)))
+ return path
+
+ def run_script(self, manifest_path, *extra):
+ argv = ["checkpoints.py", "--models", str(self.models), *extra]
+ buf = io.StringIO()
+ with mock.patch.object(sys, "argv", argv), \
+ mock.patch.object(check, "MANIFEST", str(manifest_path)), \
+ contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf):
+ code = check.main()
+ return code, buf.getvalue()
+
+ @property
+ def weights(self):
+ return self.models / "laya" / "model.safetensors"
+
+ def test_an_intact_checkpoint_passes(self):
+ self.weights.write_bytes(GOOD)
+ code, out = self.run_script(self.write_manifest())
+ self.assertEqual(code, 0, out)
+ self.assertIn("match the manifest", out)
+
+ def test_missing_file_names_the_fetch_command(self):
+ code, out = self.run_script(self.write_manifest())
+ self.assertEqual(code, 1, out)
+ self.assertIn("missing", out)
+ self.assertIn("--fetch", out)
+
+ def test_truncated_file_is_reported_with_the_refetch_remedy(self):
+ self.weights.write_bytes(GOOD[:5])
+ code, out = self.run_script(self.write_manifest())
+ self.assertEqual(code, 1, out)
+ self.assertIn("size mismatch means the file is incomplete", out)
+ self.assertIn("--fetch", out)
+ # `--record` here would write the truncation into the manifest as the new truth.
+ self.assertNotIn("re-record", out)
+
+ def test_equal_sized_digest_change_points_at_record(self):
+ self.weights.write_bytes(b"x" * len(GOOD))
+ code, out = self.run_script(self.write_manifest())
+ self.assertEqual(code, 1, out)
+ self.assertIn("review the files and re-record with", out)
+ self.assertNotIn("size mismatch means", out)
+
+ def test_fetch_repairs_a_truncated_file(self):
+ self.weights.write_bytes(GOOD[:5]) # what a killed download leaves
+ hub = FakeHub()
+ with mock.patch.dict(sys.modules, {"huggingface_hub": hub}):
+ code, out = self.run_script(self.write_manifest(), "--fetch", "--quiet")
+ self.assertEqual(code, 0, out)
+ self.assertEqual(len(hub.calls), 1, "the truncated file must be re-downloaded")
+ self.assertEqual(hub.calls[0]["revision"], "0" * 40, "at the manifest's pinned revision")
+ self.assertEqual(self.weights.read_bytes(), GOOD)
+
+ def test_fetch_leaves_a_right_sized_file_alone(self):
+ self.weights.write_bytes(GOOD)
+ hub = FakeHub()
+ with mock.patch.dict(sys.modules, {"huggingface_hub": hub}):
+ code, out = self.run_script(self.write_manifest(), "--fetch", "--quiet")
+ self.assertEqual(code, 0, out)
+ self.assertEqual(hub.calls, [], "a verifiable file needs no network call")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/verify/checkpoints.py b/verify/checkpoints.py
index 14c4691c..64b79147 100644
--- a/verify/checkpoints.py
+++ b/verify/checkpoints.py
@@ -8,7 +8,9 @@ to the recorded sha256, so a truncated download or an upstream change is caught
turning into quietly wrong answers. Exits non-zero on a missing file or a hash mismatch.
`--fetch` downloads each checkpoint at the manifest's pinned `revision`, so the hashes keep
-matching when the Hub repo moves on; without it, nothing is downloaded.
+matching when the Hub repo moves on; without it, nothing is downloaded. A file whose size does
+not match the manifest is re-downloaded by `--fetch` (a truncated transfer is repaired, not
+trusted), while a file that is present at the right size is left alone.
A mismatch is reported, not hidden: if the weights really did change, the manifest needs
re-recording (`--record`).
@@ -36,12 +38,24 @@ def sha256(path, chunk=1 << 20):
def fetch_entry(entry, revision, models_dir):
- """Download one checkpoint at `revision` into the directory its manifest path names."""
+ """Download one checkpoint at `revision` into the directory its manifest path names.
+
+ A file that is already present is only trusted when its size matches the manifest. The
+ point of `--fetch` is to end with a checkpoint the verify pass accepts, and a truncated
+ download -- the case the module docstring promises this script catches -- used to be
+ skipped by the bare existence check here, so the advice printed on failure ("fetch it
+ with --fetch") could never repair it. A wrong-sized file is removed first, because
+ `snapshot_download(local_dir=...)` resumes from its own bookkeeping and would otherwise
+ keep the file it believes it already wrote.
+ """
from huggingface_hub import snapshot_download
dest = os.path.join(models_dir, os.path.dirname(entry["path"]))
- if os.path.exists(os.path.join(dest, entry["file"])):
- return dest
+ present = os.path.join(dest, entry["file"])
+ if os.path.exists(present):
+ if os.path.getsize(present) == entry["bytes"]:
+ return dest
+ os.remove(present)
prefix = entry["subfolder"] + "/" if entry["subfolder"] else ""
snapshot_download(entry["repo"], revision=revision, local_dir=dest,
allow_patterns=[prefix + name for name in RUNTIME_FILES])
@@ -74,6 +88,7 @@ def main():
print(" fetched %-16s -> %s" % (entry["name"], dest))
problems, recorded = [], []
+ truncated, rewritten = [], []
for entry in manifest["checkpoints"]:
path = os.path.join(args.models, entry["path"])
if not os.path.exists(path):
@@ -90,6 +105,12 @@ def main():
if not ok:
problems.append("%s: %s\n expected %s (%d bytes)\n found %s (%d bytes)"
% (entry["name"], path, entry["sha256"], entry["bytes"], digest, size))
+ # The two causes need opposite remedies: a wrong size is an incomplete download,
+ # repaired by re-fetching, while identical bytes and a different hash is content
+ # that changed under a pinned revision. The old advice told both cases to
+ # `--record`, which for a truncated file means writing the truncation into the
+ # manifest as the new truth.
+ (truncated if size != entry["bytes"] else rewritten).append(entry["name"])
elif not args.quiet:
print(" OK %-16s %-34s %8.1f MB %s"
% (entry["name"], entry["encoder"] + " ctx=" + str(entry["context"]),
@@ -105,8 +126,14 @@ def main():
print("\n%d checkpoint problem(s):" % len(problems))
for p in problems:
print(" FAIL " + p)
- print("\nIf the upstream hub revision moved, re-record with: "
- ".venv/bin/python verify/checkpoints.py --record")
+ if truncated:
+ print("\nA size mismatch means the file is incomplete or was replaced (%s):"
+ "\n .venv/bin/python verify/checkpoints.py --fetch "
+ "# re-downloads at the pinned revision" % ", ".join(truncated))
+ if rewritten:
+ print("\nIf the upstream hub revision really moved (%s), review the files and "
+ "re-record with:\n .venv/bin/python verify/checkpoints.py --record"
+ % ", ".join(rewritten))
return 1
if not args.quiet:
print("\n all %d checkpoints match the manifest (hub revision %s)"