Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
3.7 KB
Raw Blame
"""Exercise corrupt tokenizer files through the native public loader."""
import json
import os
from pathlib import Path
import shlex
import shutil
import subprocess
import tempfile
import unittest

C_DIR = Path(__file__).resolve().parents[1]


class TokenizerLoadValidation(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        compiler = shlex.split(os.environ.get("CC", "cc"))
        if not compiler or shutil.which(compiler[0]) is None:
            raise unittest.SkipTest("a C compiler is required for the native loader")
        cls.tmp = tempfile.TemporaryDirectory()
        cls.addClassCleanup(cls.tmp.cleanup)
        cls.root = Path(cls.tmp.name)
        source = cls.root / "load.c"
        source.write_text('#include "tok.h"\nint main(int argc, char **argv) {\n'
                          '  if (argc != 2) return 2;\n'
                          '  Tok t; tok_load(&t, argv[1]); tok_free(&t); return 0;\n}\n')
        cls.binary = cls.root / ("load.exe" if os.name == "nt" else "load")
        subprocess.run(compiler + ["-O1", "-Wall", "-Wextra", "-Wno-unused-function",
                                   "-I", str(C_DIR), str(source), "-o", str(cls.binary)],
                       check=True, capture_output=True, text=True)

    def load(self, payload):
        path = self.root / "tokenizer.json"
        path.write_bytes(payload.encode() if isinstance(payload, str) else payload)
        return subprocess.run([str(self.binary), str(path)], capture_output=True,
                              text=True, timeout=10)

    def reject(self, payload):
        result = self.load(payload)
        self.assertEqual(result.returncode, 1, result.stderr)
        self.assertIn("tokenizer.json:", result.stderr)

    def test_invalid_container_types(self):
        for field, value in (("vocab", [0]), ("vocab", "abc"),
                             ("merges", {"a b": 0}), ("added_tokens", {"a": 0})):
            with self.subTest(field=field, value=value):
                document = {"model": {"vocab": {"a": 0}, "merges": []}}
                if field == "added_tokens":
                    document[field] = value
                else:
                    document["model"][field] = value
                self.reject(json.dumps(document))

    def test_vocab_and_added_ids_must_be_bounded_integers(self):
        for value in ("-0.5", "0.5", "1e100", "1e309", "2097153", "-1", '"0"'):
            with self.subTest(value=value):
                self.reject('{"model":{"vocab":{"a":' + value + '}}}')
                self.reject('{"model":{"vocab":{"a":0}},'
                            '"added_tokens":[{"content":"<s>","id":' + value + '}]}')

    def test_partial_or_malformed_json_is_rejected(self):
        prefix = '{"model":{"vocab":{"a":0}}'
        for payload in (prefix, prefix + ',}', prefix + '} trailing',
                        (prefix + '}').encode() + b'\0garbage'):
            with self.subTest(payload=payload):
                self.reject(payload)

    def test_valid_sparse_ids_and_integral_exponents(self):
        for value in ("0", "2e0", "3.0"):
            with self.subTest(value=value):
                result = self.load('{"model":{"vocab":{"a":' + value + '},"merges":[]},'
                                   '"added_tokens":[{"content":"<s>","id":8,"special":true}]}')
                self.assertEqual(result.returncode, 0, result.stderr)

    def test_merge_forms_remain_supported(self):
        for merge in (["a", "b"], "a b"):
            with self.subTest(merge=merge):
                result = self.load(json.dumps({"model": {"vocab": {"a": 0, "b": 1, "ab": 2},
                                                         "merges": [merge]}}))
                self.assertEqual(result.returncode, 0, result.stderr)


if __name__ == "__main__":
    unittest.main()