Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(convert): publish resumable shards atomically

Blake345bc6c8953b
diff --git a/c/tests/test_converter_atomic_output.py b/c/tests/test_converter_atomic_output.py
new file mode 100644
index 00000000..96222153
--- /dev/null
+++ b/c/tests/test_converter_atomic_output.py
@@ -0,0 +1,66 @@
+import importlib.util
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+CONVERTER = Path(__file__).resolve().parent.parent / "tools" / "convert_fp8_to_int4.py"
+
+
+def load_atomic_save():
+ spec = importlib.util.spec_from_file_location("atomic_converter_test", CONVERTER)
+ module = importlib.util.module_from_spec(spec)
+ with mock.patch.dict(sys.modules, {"numpy": types.ModuleType("numpy")}):
+ spec.loader.exec_module(module)
+ return module._save_file_atomic
+
+
+_save_file_atomic = load_atomic_save()
+
+
+class AtomicConverterOutputTest(unittest.TestCase):
+ def setUp(self):
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.destination = Path(self.directory.name) / "out-00000.safetensors"
+ self.temporary = Path(str(self.destination) + ".tmp")
+
+ def test_success_publishes_complete_output(self):
+ def save_file(tensors, path):
+ Path(path).write_bytes(tensors["payload"])
+
+ _save_file_atomic(save_file, {"payload": b"complete"}, self.destination)
+
+ self.assertEqual(self.destination.read_bytes(), b"complete")
+ self.assertFalse(self.temporary.exists())
+
+ def test_failed_save_leaves_no_resumable_output(self):
+ def save_file(tensors, path):
+ Path(path).write_bytes(b"partial")
+ raise OSError("disk full")
+
+ with self.assertRaisesRegex(OSError, "disk full"):
+ _save_file_atomic(save_file, {}, self.destination)
+
+ self.assertFalse(self.destination.exists())
+ self.assertFalse(self.temporary.exists())
+
+ def test_failed_overwrite_preserves_previous_output(self):
+ self.destination.write_bytes(b"previous")
+
+ def save_file(tensors, path):
+ Path(path).write_bytes(b"partial")
+ raise KeyboardInterrupt
+
+ with self.assertRaises(KeyboardInterrupt):
+ _save_file_atomic(save_file, {}, self.destination)
+
+ self.assertEqual(self.destination.read_bytes(), b"previous")
+ self.assertFalse(self.temporary.exists())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index ab55403b..6b9c9a1d 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -32,6 +32,23 @@ import numpy as np
_positioned_write_lock = threading.Lock()
+def _save_file_atomic(save_file, tensors, destination, **kwargs):
+ destination = os.fspath(destination)
+ temporary = destination + ".tmp"
+ try:
+ os.remove(temporary)
+ except FileNotFoundError:
+ pass
+ try:
+ save_file(tensors, temporary, **kwargs)
+ os.replace(temporary, destination)
+ finally:
+ try:
+ os.remove(temporary)
+ except OSError:
+ pass
+
+
def _positioned_write(fd, data, offset):
remaining = memoryview(data)
pwrite = getattr(os, "pwrite", None)
@@ -716,7 +733,7 @@ def main():
done[key] = ""
else:
name = f"{prefix}{n:05d}.safetensors"
- save_file(out, os.path.join(a.outdir, name))
+ _save_file_atomic(save_file, out, os.path.join(a.outdir, name))
done[key] = name; n += 1; fresh += 1
tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto
with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest
@@ -986,7 +1003,7 @@ def main():
print(f"[MTP {i+1}/{len(mtp_shards)}] downloading {sh}...", flush=True)
p = download_retry(a.repo, sh, tmp)
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_mtp=True, group_size=a.group_size, bits_map=bits_map)
- save_file(out, outp)
+ _save_file_atomic(save_file, out, outp)
os.remove(p)
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
if os.path.isfile(blob): os.remove(blob)
@@ -1010,7 +1027,7 @@ def main():
print(f"[IDX {i+1}/{len(idx_shards)}] downloading {sh}...", flush=True)
p = download_retry(a.repo, sh, tmp)
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_idx=True, group_size=a.group_size, bits_map=bits_map)
- if out: save_file(out, outp)
+ if out: _save_file_atomic(save_file, out, outp)
os.remove(p)
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
if os.path.isfile(blob): os.remove(blob)
@@ -1028,7 +1045,7 @@ def main():
print(f"[{i+1}/{len(shards)}] downloading {sh} ({free_gb(a.outdir):.0f} GB free)...", flush=True)
p = download_retry(a.repo, sh, tmp)
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map)
- save_file(out, outp)
+ _save_file_atomic(save_file, out, outp)
os.remove(p) # <-- cancella subito lo shard fp8
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
if os.path.isfile(blob): os.remove(blob)