Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
"""Resumable downloads, against a local fake of the Hugging Face hub.
Nothing here touches the network: tests/fake_hub.py serves a repository from a
temporary folder with the hub's listing API, a redirect to a "CDN" path, Range
support and injected faults. The contract under test: a dropped connection
resumes from the bytes on disk (Range, then append), a server that ignores
Range restarts the file instead of appending a second copy, corrupt bytes never
become a file without `.part`, and the token never follows the redirect.
"""
import hashlib
import os
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(HERE))
import setup_download # noqa: E402
from fake_hub import FakeHub # noqa: E402
REPO = "tester/tiny-model"
def make_repo(root):
files = {
"config.json": b'{"model_type": "qwen3_5_moe_text"}',
"tokenizer.json": b'{"model": {}}' * 10,
"model-00000.safetensors": os.urandom(300_000),
"model-00001.safetensors": os.urandom(200_000),
"sub/extra.bin": os.urandom(5_000),
"README.md": b"# readme\n",
}
for name, data in files.items():
path = Path(root, *name.split("/"))
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
return files
class DownloadTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.src = os.path.join(self.tmp.name, "src")
self.dst = os.path.join(self.tmp.name, "dst")
self.files = make_repo(self.src)
self.hub = FakeHub({REPO: self.src})
self.hub.__enter__()
self.addCleanup(self.hub.__exit__)
# Retries without the real backoff sleeps.
patcher = mock.patch.object(setup_download.time, "sleep")
patcher.start()
self.addCleanup(patcher.stop)
def listing(self, **kw):
return setup_download.list_repo_files(REPO, "main", base=self.hub.base, **kw)
def spec(self, name):
return next(f for f in self.listing() if f["path"] == name)
def url(self, name):
return setup_download.file_url(REPO, "main", name, self.hub.base)
def ranges(self, name):
return [r["headers"].get("Range") for r in self.hub.requests
if r["path"].endswith("/" + name) and "/cdn/" in r["path"]]
def test_listing_sizes_hashes_and_filters(self):
files = {f["path"]: f for f in self.listing()}
self.assertEqual(set(files), set(self.files))
shard = files["model-00000.safetensors"]
self.assertEqual(shard["size"], 300_000)
self.assertEqual(shard["sha256"], hashlib.sha256(self.files["model-00000.safetensors"]).hexdigest())
self.assertIsNone(files["config.json"]["sha256"])
self.assertIsNotNone(files["config.json"]["git_oid"])
kept = {f["path"] for f in self.listing(exclude=("sub/*", "*.md"))}
self.assertEqual(kept, {"config.json", "tokenizer.json", "model-00000.safetensors",
"model-00001.safetensors"})
only = {f["path"] for f in self.listing(include=("*.safetensors",))}
self.assertEqual(only, {"model-00000.safetensors", "model-00001.safetensors"})
def test_listing_follows_pagination(self):
self.hub.page_size = 2
self.assertEqual({f["path"] for f in self.listing()}, set(self.files))
def test_missing_repo_is_a_clear_error(self):
with self.assertRaises(setup_download.DownloadError) as caught:
setup_download.list_repo_files("nobody/nothing", base=self.hub.base)
self.assertIn("not found", str(caught.exception))
def test_whole_repo_and_manifest(self):
files = self.listing()
events = []
setup_download.download_repo(REPO, "main", self.dst, files, base=self.hub.base,
progress=events.append)
for name, data in self.files.items():
self.assertEqual(Path(self.dst, *name.split("/")).read_bytes(), data)
self.assertTrue(setup_download.is_complete(self.dst))
self.assertEqual(events[-1]["done"], events[-1]["total"])
self.assertEqual(events[0]["file"], "README.md") # small files first
# A rerun fetches nothing.
before = len(self.hub.requests)
setup_download.download_repo(REPO, "main", self.dst, files, base=self.hub.base)
self.assertEqual(len(self.hub.requests), before)
def test_empty_repository_file_is_completed_and_manifest_is_reusable(self):
Path(self.src, "empty.txt").write_bytes(b"")
files = self.listing(include=("empty.txt",))
self.assertEqual(files[0]["git_oid"], hashlib.sha1(b"blob 0\0").hexdigest())
setup_download.download_repo(REPO, "main", self.dst, files, base=self.hub.base)
self.assertEqual(Path(self.dst, "empty.txt").read_bytes(), b"")
self.assertFalse(Path(self.dst, "empty.txt.part").exists())
self.assertTrue(setup_download.is_complete(self.dst))
before = len(self.hub.requests)
setup_download.download_repo(REPO, "main", self.dst, files, base=self.hub.base)
self.assertEqual(len(self.hub.requests), before)
def test_interrupted_download_resumes_with_range(self):
name = "model-00000.safetensors"
spec = self.spec(name)
dest = os.path.join(self.dst, name)
self.hub.cut_after[name] = 120_000
with self.assertRaises(setup_download.DownloadError) as caught:
setup_download.download_file(self.url(name), dest, spec, retries=0)
self.assertIn("rerun to continue", str(caught.exception))
self.assertFalse(os.path.exists(dest))
self.assertEqual(os.path.getsize(dest + ".part"), 120_000)
# The rerun: asks for the rest only, and the file checks out.
result = setup_download.download_file(self.url(name), dest, spec, retries=0)
self.assertEqual(result, "downloaded")
self.assertEqual(Path(dest).read_bytes(), self.files[name])
self.assertEqual(self.ranges(name)[-1], "bytes=120000-")
self.assertFalse(os.path.exists(dest + ".part"))
def test_dropped_connection_retries_inside_one_call(self):
name = "model-00001.safetensors"
self.hub.cut_after[name] = 50_000
dest = os.path.join(self.dst, name)
setup_download.download_file(self.url(name), dest, self.spec(name), retries=2)
self.assertEqual(Path(dest).read_bytes(), self.files[name])
self.assertEqual(self.ranges(name), [None, "bytes=50000-"])
def test_server_ignoring_range_restarts_instead_of_appending(self):
name = "model-00001.safetensors"
dest = os.path.join(self.dst, name)
os.makedirs(self.dst, exist_ok=True)
Path(dest + ".part").write_bytes(self.files[name][:70_000])
self.hub.ignore_range = True
setup_download.download_file(self.url(name), dest, self.spec(name), retries=0)
self.assertEqual(Path(dest).read_bytes(), self.files[name])
def test_corrupt_partial_is_removed_not_kept(self):
name = "model-00000.safetensors"
dest = os.path.join(self.dst, name)
os.makedirs(self.dst, exist_ok=True)
Path(dest + ".part").write_bytes(b"\0" * 100_000) # wrong bytes, right offset
with self.assertRaises(setup_download.DownloadError) as caught:
setup_download.download_file(self.url(name), dest, self.spec(name), retries=0)
self.assertIn("checksum", str(caught.exception))
self.assertFalse(os.path.exists(dest))
self.assertFalse(os.path.exists(dest + ".part"))
# and the rerun after that starts clean and succeeds
setup_download.download_file(self.url(name), dest, self.spec(name), retries=0)
self.assertEqual(Path(dest).read_bytes(), self.files[name])
def test_oversized_partial_starts_over(self):
name = "config.json"
dest = os.path.join(self.dst, name)
os.makedirs(self.dst, exist_ok=True)
Path(dest + ".part").write_bytes(b"x" * 10_000)
setup_download.download_file(self.url(name), dest, self.spec(name), retries=0)
self.assertEqual(Path(dest).read_bytes(), self.files[name])
def test_token_reaches_the_hub_and_not_the_cdn(self):
name = "config.json"
setup_download.download_file(self.url(name), os.path.join(self.dst, name),
self.spec(name), token="hf_secret", retries=0)
hub_side = [r for r in self.hub.requests if "/resolve/" in r["path"]]
cdn_side = [r for r in self.hub.requests if "/cdn/" in r["path"]]
self.assertEqual(hub_side[-1]["headers"].get("Authorization"), "Bearer hf_secret")
self.assertIsNone(cdn_side[-1]["headers"].get("Authorization"))
def test_bytes_present_counts_partials(self):
files = self.listing()
os.makedirs(self.dst, exist_ok=True)
Path(self.dst, "config.json").write_bytes(self.files["config.json"])
Path(self.dst, "model-00000.safetensors.part").write_bytes(b"x" * 1000)
self.assertEqual(setup_download.bytes_present(self.dst, files),
len(self.files["config.json"]) + 1000)
self.assertFalse(setup_download.is_complete(self.dst))
def test_repository_download_refuses_existing_symlinks_outside_model_dir(self):
os.makedirs(self.dst)
for index, (linked_path, name) in enumerate(((Path(self.dst, "sub"), "sub/extra.bin"),
(Path(self.dst, "model-00000.safetensors.part"), "model-00000.safetensors"))):
with self.subTest(path=name):
outside = Path(self.tmp.name, f"outside-{index}")
outside.mkdir()
target = outside if name.startswith("sub/") else outside / "part.bin"
if not name.startswith("sub/"):
target.write_bytes(b"")
try:
linked_path.symlink_to(target, target_is_directory=name.startswith("sub/"))
except NotImplementedError as error:
self.skipTest(f"symlinks unavailable: {error}")
except OSError as error:
if os.name == "nt" and getattr(error, "winerror", None) == 1314:
self.skipTest(f"symlink privilege unavailable: {error}")
raise
with self.assertRaises(setup_download.DownloadError):
setup_download.download_repo(REPO, "main", self.dst, [self.spec(name)], base=self.hub.base)
self.assertFalse((outside / "extra.bin").exists())
if not name.startswith("sub/"):
self.assertEqual(target.read_bytes(), b"")
def test_repository_manifest_refuses_existing_symlinks_outside_model_dir(self):
os.makedirs(self.dst)
for index, name in enumerate((setup_download.MANIFEST, setup_download.MANIFEST + ".tmp")):
with self.subTest(path=name):
outside = Path(self.tmp.name, f"outside-manifest-{index}.json")
outside.write_bytes(b"keep me")
linked = Path(self.dst, name)
try:
linked.symlink_to(outside)
except NotImplementedError as error:
self.skipTest(f"symlinks unavailable: {error}")
except OSError as error:
if os.name == "nt" and getattr(error, "winerror", None) == 1314:
self.skipTest(f"symlink privilege unavailable: {error}")
raise
try:
with self.assertRaises(setup_download.DownloadError):
setup_download.download_repo(REPO, "main", self.dst, [self.spec("README.md")], base=self.hub.base)
self.assertEqual(outside.read_bytes(), b"keep me")
finally:
linked.unlink()
def test_unsafe_paths_are_refused(self):
for bad in ("../escape.bin", "a/../../b", "C:/x", ""):
with self.subTest(path=bad), self.assertRaises(setup_download.DownloadError):
setup_download.safe_join(self.dst, bad)
def test_throughput_probe(self):
rate = setup_download.probe_throughput(self.url("model-00000.safetensors"),
max_bytes=100_000, max_seconds=5)
self.assertGreater(rate, 0)
self.assertEqual(self.ranges("model-00000.safetensors")[-1], "bytes=0-99999")
def test_curl_speed_parsing(self):
self.assertEqual(setup_download.parse_curl_speed("3456789.000"), 3456789.0)
self.assertEqual(setup_download.parse_curl_speed("3456789,500"), 3456789.5)
self.assertIsNone(setup_download.parse_curl_speed(""))
def test_verify_file(self):
name = "model-00001.safetensors"
path = Path(self.dst, name)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(self.files[name])
self.assertTrue(setup_download.verify_file(str(path), self.spec(name)))
path.write_bytes(b"\1" * len(self.files[name]))
self.assertFalse(setup_download.verify_file(str(path), self.spec(name)))
def stand_in_curl(self):
"""A curl.exe that does what the transport asks of it: fetch the last
argument into the file after -o. A script with a shebang on POSIX, a
.cmd around the same script on Windows."""
script = Path(self.tmp.name, "curl_stand_in.py")
script.write_text(
"import sys, urllib.request\n"
"args = sys.argv[1:]\n"
"out = args[args.index('-o') + 1]\n"
"data = urllib.request.urlopen(args[-1]).read()\n"
"open(out, 'wb').write(data)\n")
if os.name == "nt":
wrapper = Path(self.tmp.name, "curl.cmd")
wrapper.write_text(f'@"{sys.executable}" "{script}" %*\r\n')
return str(wrapper)
wrapper = Path(self.tmp.name, "curl.exe")
wrapper.write_text("#!" + sys.executable + "\n" + script.read_text())
wrapper.chmod(0o755)
return str(wrapper)
def test_windows_curl_transport(self):
"""WSL's slow-network path: Windows' curl.exe writes the .part file, then
the same size and hash checks decide whether it becomes the file."""
fake_curl = self.stand_in_curl()
name = "model-00000.safetensors"
dest = os.path.join(self.dst, name)
seen = []
with mock.patch.object(setup_download, "wsl_windows_path", side_effect=lambda p: p):
result = setup_download.download_file_windows(
self.url(name), dest, self.spec(name), fake_curl,
progress=lambda done, total: seen.append((done, total)))
self.assertEqual(result, "downloaded")
self.assertEqual(Path(dest).read_bytes(), self.files[name])
self.assertEqual(seen[-1], (300_000, 300_000))
# A transfer that ends with the wrong bytes is not kept.
other = "model-00001.safetensors"
bad = dict(self.spec(other), sha256="0" * 64)
with mock.patch.object(setup_download, "wsl_windows_path", side_effect=lambda p: p), \
self.assertRaises(setup_download.DownloadError):
setup_download.download_file_windows(self.url(other), os.path.join(self.dst, other),
bad, fake_curl)
self.assertFalse(os.path.exists(os.path.join(self.dst, other)))
def test_token_from_file(self):
with tempfile.TemporaryDirectory() as hf_home:
Path(hf_home, "token").write_text("hf_from_file\n")
env = {k: v for k, v in os.environ.items()
if k not in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN")}
env["HF_HOME"] = hf_home
with mock.patch.dict(os.environ, env, clear=True):
self.assertEqual(setup_download.hf_token(), "hf_from_file")
if __name__ == "__main__":
unittest.main()