Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(verify): checkpoints.py repairs a truncated download instead of trusting it
Aashish4df855ef01c6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 39843b20..e03820d6 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -115,6 +115,7 @@ jobs:python tests/test_onnx_long.pypython tests/test_onnx_sort.pypython tests/test_download.py+ python tests/test_verify_checkpoints.pypython tests/test_shortlist.pypython tests/test_decision_model.pypython tests/test_head_checkpointing.py@@ -300,7 +301,7 @@ jobs:run: |for t in test_router test_criteria test_batch test_predict_long \test_structured test_structured_api test_onnx_lang_parity \- test_download test_shortlist test_decision_model \+ test_download test_verify_checkpoints test_shortlist test_decision_model \test_head_checkpointing test_packaging test_doc_tables \test_tokenizer_cache test_tokenizer_concurrency test_lazy_import \test_runtime_fixes test_lang_guess test_identifier_complexity \diff --git a/tests/test_verify_checkpoints.py b/tests/test_verify_checkpoints.pynew file mode 100644index 00000000..514722bc--- /dev/null+++ b/tests/test_verify_checkpoints.py@@ -0,0 +1,144 @@+"""`verify/checkpoints.py`: what `--fetch` repairs, and which remedy the failure banner offers.++The script promises the manifest catches "a truncated download or an upstream change ...+rather than turning into quietly wrong answers". Two defects stood between a failure and its+fix, both about the advice a user follows after one:++1. `--fetch` skipped any checkpoint whose file existed, so the banner's instruction -- "fetch+ it with `--fetch`" -- could never repair a truncated transfer. It now re-downloads when+ the size disagrees with the manifest, and leaves a right-sized file alone (no network call).+2. The banner offered one remedy for both causes: `--record`. For a truncated file that+ writes the truncation into the manifest as the new truth. It now separates the two: a size+ mismatch is re-fetched, an equal-sized digest change is the one to review and re-record.++No network and no real weights: the manifest is rewritten into a temp directory and+`huggingface_hub.snapshot_download` is a stub that writes the bytes the manifest expects.++Run: python tests/test_verify_checkpoints.py+"""+import contextlib+import hashlib+import importlib.util+import io+import json+import os+from pathlib import Path+import sys+import tempfile+from unittest import mock+import unittest++ROOT = Path(__file__).resolve().parents[1]+SCRIPT = ROOT / "verify" / "checkpoints.py"++spec = importlib.util.spec_from_file_location("verify_checkpoints_under_test", SCRIPT)+check = importlib.util.module_from_spec(spec)+spec.loader.exec_module(check)++GOOD = b"the checkpoint bytes, as fetched at the pinned revision"+++def manifest_for(relpath, name="english", good=GOOD):+ return {+ "repo": "convaiinnovations/laya",+ "revision": "0" * 40,+ "note": "fixture",+ "checkpoints": [{+ "name": name, "repo": "convaiinnovations/laya", "subfolder": None,+ "path": relpath, "file": "model.safetensors",+ "bytes": len(good), "sha256": hashlib.sha256(good).hexdigest(),+ "params": 1, "encoder": "fixture", "context": 8,+ }],+ }+++class FakeHub:+ """`huggingface_hub` stand-in: remembers the calls and writes the expected bytes."""++ def __init__(self, payload=GOOD):+ self.payload = payload+ self.calls = []++ def snapshot_download(self, repo, revision=None, local_dir=None, allow_patterns=None):+ self.calls.append({"repo": repo, "revision": revision})+ os.makedirs(local_dir, exist_ok=True)+ with open(os.path.join(local_dir, "model.safetensors"), "wb") as handle:+ handle.write(self.payload)+ return local_dir+++class CheckpointsTests(unittest.TestCase):+ def setUp(self):+ self._tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self._tmp.cleanup)+ self.root = Path(self._tmp.name)+ self.models = self.root / "models"+ (self.models / "laya").mkdir(parents=True)++ def write_manifest(self, **kwargs):+ path = self.root / "checkpoints.json"+ path.write_text(json.dumps(manifest_for("laya/model.safetensors", **kwargs)))+ return path++ def run_script(self, manifest_path, *extra):+ argv = ["checkpoints.py", "--models", str(self.models), *extra]+ buf = io.StringIO()+ with mock.patch.object(sys, "argv", argv), \+ mock.patch.object(check, "MANIFEST", str(manifest_path)), \+ contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf):+ code = check.main()+ return code, buf.getvalue()++ @property+ def weights(self):+ return self.models / "laya" / "model.safetensors"++ def test_an_intact_checkpoint_passes(self):+ self.weights.write_bytes(GOOD)+ code, out = self.run_script(self.write_manifest())+ self.assertEqual(code, 0, out)+ self.assertIn("match the manifest", out)++ def test_missing_file_names_the_fetch_command(self):+ code, out = self.run_script(self.write_manifest())+ self.assertEqual(code, 1, out)+ self.assertIn("missing", out)+ self.assertIn("--fetch", out)++ def test_truncated_file_is_reported_with_the_refetch_remedy(self):+ self.weights.write_bytes(GOOD[:5])+ code, out = self.run_script(self.write_manifest())+ self.assertEqual(code, 1, out)+ self.assertIn("size mismatch means the file is incomplete", out)+ self.assertIn("--fetch", out)+ # `--record` here would write the truncation into the manifest as the new truth.+ self.assertNotIn("re-record", out)++ def test_equal_sized_digest_change_points_at_record(self):+ self.weights.write_bytes(b"x" * len(GOOD))+ code, out = self.run_script(self.write_manifest())+ self.assertEqual(code, 1, out)+ self.assertIn("review the files and re-record with", out)+ self.assertNotIn("size mismatch means", out)++ def test_fetch_repairs_a_truncated_file(self):+ self.weights.write_bytes(GOOD[:5]) # what a killed download leaves+ hub = FakeHub()+ with mock.patch.dict(sys.modules, {"huggingface_hub": hub}):+ code, out = self.run_script(self.write_manifest(), "--fetch", "--quiet")+ self.assertEqual(code, 0, out)+ self.assertEqual(len(hub.calls), 1, "the truncated file must be re-downloaded")+ self.assertEqual(hub.calls[0]["revision"], "0" * 40, "at the manifest's pinned revision")+ self.assertEqual(self.weights.read_bytes(), GOOD)++ def test_fetch_leaves_a_right_sized_file_alone(self):+ self.weights.write_bytes(GOOD)+ hub = FakeHub()+ with mock.patch.dict(sys.modules, {"huggingface_hub": hub}):+ code, out = self.run_script(self.write_manifest(), "--fetch", "--quiet")+ self.assertEqual(code, 0, out)+ self.assertEqual(hub.calls, [], "a verifiable file needs no network call")+++if __name__ == "__main__":+ unittest.main()diff --git a/verify/checkpoints.py b/verify/checkpoints.pyindex 14c4691c..64b79147 100644--- a/verify/checkpoints.py+++ b/verify/checkpoints.py@@ -8,7 +8,9 @@ to the recorded sha256, so a truncated download or an upstream change is caughtturning into quietly wrong answers. Exits non-zero on a missing file or a hash mismatch.`--fetch` downloads each checkpoint at the manifest's pinned `revision`, so the hashes keep-matching when the Hub repo moves on; without it, nothing is downloaded.+matching when the Hub repo moves on; without it, nothing is downloaded. A file whose size does+not match the manifest is re-downloaded by `--fetch` (a truncated transfer is repaired, not+trusted), while a file that is present at the right size is left alone.A mismatch is reported, not hidden: if the weights really did change, the manifest needsre-recording (`--record`).@@ -36,12 +38,24 @@ def sha256(path, chunk=1 << 20):def fetch_entry(entry, revision, models_dir):- """Download one checkpoint at `revision` into the directory its manifest path names."""+ """Download one checkpoint at `revision` into the directory its manifest path names.++ A file that is already present is only trusted when its size matches the manifest. The+ point of `--fetch` is to end with a checkpoint the verify pass accepts, and a truncated+ download -- the case the module docstring promises this script catches -- used to be+ skipped by the bare existence check here, so the advice printed on failure ("fetch it+ with --fetch") could never repair it. A wrong-sized file is removed first, because+ `snapshot_download(local_dir=...)` resumes from its own bookkeeping and would otherwise+ keep the file it believes it already wrote.+ """from huggingface_hub import snapshot_downloaddest = os.path.join(models_dir, os.path.dirname(entry["path"]))- if os.path.exists(os.path.join(dest, entry["file"])):- return dest+ present = os.path.join(dest, entry["file"])+ if os.path.exists(present):+ if os.path.getsize(present) == entry["bytes"]:+ return dest+ os.remove(present)prefix = entry["subfolder"] + "/" if entry["subfolder"] else ""snapshot_download(entry["repo"], revision=revision, local_dir=dest,allow_patterns=[prefix + name for name in RUNTIME_FILES])@@ -74,6 +88,7 @@ def main():print(" fetched %-16s -> %s" % (entry["name"], dest))problems, recorded = [], []+ truncated, rewritten = [], []for entry in manifest["checkpoints"]:path = os.path.join(args.models, entry["path"])if not os.path.exists(path):@@ -90,6 +105,12 @@ def main():if not ok:problems.append("%s: %s\n expected %s (%d bytes)\n found %s (%d bytes)"% (entry["name"], path, entry["sha256"], entry["bytes"], digest, size))+ # The two causes need opposite remedies: a wrong size is an incomplete download,+ # repaired by re-fetching, while identical bytes and a different hash is content+ # that changed under a pinned revision. The old advice told both cases to+ # `--record`, which for a truncated file means writing the truncation into the+ # manifest as the new truth.+ (truncated if size != entry["bytes"] else rewritten).append(entry["name"])elif not args.quiet:print(" OK %-16s %-34s %8.1f MB %s"% (entry["name"], entry["encoder"] + " ctx=" + str(entry["context"]),@@ -105,8 +126,14 @@ def main():print("\n%d checkpoint problem(s):" % len(problems))for p in problems:print(" FAIL " + p)- print("\nIf the upstream hub revision moved, re-record with: "- ".venv/bin/python verify/checkpoints.py --record")+ if truncated:+ print("\nA size mismatch means the file is incomplete or was replaced (%s):"+ "\n .venv/bin/python verify/checkpoints.py --fetch "+ "# re-downloads at the pinned revision" % ", ".join(truncated))+ if rewritten:+ print("\nIf the upstream hub revision really moved (%s), review the files and "+ "re-record with:\n .venv/bin/python verify/checkpoints.py --record"+ % ", ".join(rewritten))return 1if not args.quiet:print("\n all %d checkpoints match the manifest (hub revision %s)"