Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(inkling): the dense int4 converter could not finish on Windows
kevin93272f7bc9dfadf5
diff --git a/c/tests/test_convert_inkling_dense_portable.py b/c/tests/test_convert_inkling_dense_portable.pynew file mode 100644index 00000000..70077464--- /dev/null+++ b/c/tests/test_convert_inkling_dense_portable.py@@ -0,0 +1,82 @@+"""tools/convert_inkling_dense_int4.py must finish on Windows and on a non-UTF-8 console.++docs/inkling.md sends a RAM-constrained user to this converter. Two things+stopped it from ever writing the container outside a UTF-8 POSIX shell:++ * it measured free space with os.statvfs, which does not exist on Windows,+ so every Windows run died with AttributeError right after the plan, before+ a byte was written;+ * its progress lines print "β¦" and the success line "β ". Where stdout cannot+ encode them (cp949/cp932 cannot encode "β ", the C locale neither), the+ print raised UnicodeEncodeError: at the first line in the C locale, and on+ a CJK Windows console after the container was written, so the run exited 1+ and never showed the quantization-error report.++The test converts a tiny shard in a child whose stdout is not UTF-8+(PYTHONUTF8=0 and the C locale, or the Windows ANSI code page).+"""+import json+import os+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++try:+ import numpy as np+ HAVE_NUMPY = True+except ImportError:+ HAVE_NUMPY = False++TOOL = Path(__file__).resolve().parent.parent / "tools" / "convert_inkling_dense_int4.py"++CHILD_ENV = {**os.environ, "PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0",+ "LC_ALL": "C", "LANG": "C"}+CHILD_ENV.pop("PYTHONIOENCODING", None)+++def write_shard(path, tensors):+ header, blobs, offset = {}, [], 0+ for name, array in tensors.items():+ data = array.astype(np.float32).tobytes()+ header[name] = {"dtype": "F32", "shape": list(array.shape),+ "data_offsets": [offset, offset + len(data)]}+ blobs.append(data)+ offset += len(data)+ encoded = json.dumps(header).encode()+ with open(path, "wb") as stream:+ stream.write(struct.pack("<Q", len(encoded)))+ stream.write(encoded)+ for data in blobs:+ stream.write(data)+++def read_header(path):+ with open(path, "rb") as stream:+ size = struct.unpack("<Q", stream.read(8))[0]+ return json.loads(stream.read(size))++[email protected](HAVE_NUMPY, "numpy is not installed; the converter imports it")+class ConvertInklingDensePortableTest(unittest.TestCase):+ def test_converts_and_reports_in_a_non_utf8_locale(self):+ rng = np.random.default_rng(0)+ with tempfile.TemporaryDirectory() as model:+ write_shard(Path(model, "model-00001.safetensors"), {+ "model.embed_tokens.weight": rng.standard_normal((8, 64)),+ "model.layers.0.input_layernorm.weight": np.ones(64),+ })+ result = subprocess.run([sys.executable, str(TOOL), "--dir", model],+ env=CHILD_ENV, capture_output=True, timeout=120)+ output = (result.stdout + result.stderr).decode("utf-8", "replace")+ self.assertEqual(result.returncode, 0, output[-3000:])+ header = read_header(Path(model, "dense-int4g64.safetensors"))+ self.assertEqual(header["__metadata__"]["model.embed_tokens.weight"], "fmt=1;orig=8x64")+ self.assertEqual(header["__metadata__"]["model.layers.0.input_layernorm.weight"], "fmt=raw")+ self.assertIn("embed/head", output)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/convert_inkling_dense_int4.py b/c/tools/convert_inkling_dense_int4.pyindex 4e6a27a5..10221b47 100644--- a/c/tools/convert_inkling_dense_int4.py+++ b/c/tools/convert_inkling_dense_int4.py@@ -37,9 +37,19 @@ Uso:/path/al/modello/dense-int4g64/dense.safetensorsIl motore trova il container da solo (INK_DENSE_Q4=0 lo ignora)."""-import argparse, json, os, re, struct, sys, time+import argparse, json, os, re, shutil, struct, sys, timeimport numpy as np+# The progress lines print "β¦" and the success line "β ", which a stdout in+# cp949 or cp932 (a Korean or Japanese Windows console) or the C locale cannot+# encode: the print raised and the run exited 1, after writing the container,+# without the quantization-error report. Escape what the encoding cannot hold,+# as CPython already does on stderr; a UTF-8 stdout is unaffected.+try:+ sys.stdout.reconfigure(errors="backslashreplace")+except AttributeError:+ pass+DIR = os.environ.get("INKLING_DIR", ".")OUT = os.path.join(DIR, "dense-int4g64.safetensors")GS = 64@@ -203,7 +213,7 @@ def main():print(f" densa da convertire: {len(plan)} tensori {by}")print(f" input bf16/f32 : {tot_in/1e9:7.2f} GB")print(f" output stimato : {tot_out/1e9:7.2f} GB ({100*tot_out/max(tot_in,1):.0f}%)")- free = os.statvfs(DIR).f_bavail * os.statvfs(DIR).f_frsize+ free = shutil.disk_usage(DIR).free # os.statvfs does not exist on Windowsprint(f" spazio libero : {free/1e9:7.2f} GB")if a.plan:print("\n[--plan] nessuna scrittura. Rilancia senza --plan per convertire.")