Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

52.9 KB
Raw Blame
"""Patch an FTW checkpoint written by an older FreeToken so the current build loads it.

    python scripts/ftw_hotfix.py --ftw <ftw_dir> [--out <new_dir>] \
        [--repo <hf_repo_id> [--revision <rev>] | --source <local_hf_dir>] [--dry-run]

The tool builds the current model on the meta device from the FTW's own config.json, diffs the
FTW index against the tensors that model declares, and repairs the differences:

- renames (DeepSeek-V4: the dense tree moved under ``model.`` and ``.scale`` became ``.weight_scale_inv``)
- fp8 weights the current model declares as bf16 (old runtime fp8 residue) are dequantized, their ``.weight_scale`` dropped
- tensors the model declares but the FTW lacks (``input_scale``) are fetched from the HF repo by byte range
- router selection biases an older build rounded to bf16 (GLM-5.2) are restored in fp32 from the source checkpoint
- an FTW converted before its family served images gets its vision encoder: the tower's checkpoint tensors are fetched into a scratch checkpoint dir and the family's own weight reader emits them under the names the FTW needs
- a Qwen3.8-Flash-Next FTW without the PLE n-gram table gets it written as ``ple-table-*.safetensors``

Before anything is written the FTW is checked against the model: shapes, dtypes, byte counts and
shard files must agree, or the layout is reported as unsupported.

In place, new tensors go into an appended shard and the index is swapped atomically; shards left
with unreferenced bytes are then compacted one at a time, each step ending in another atomic index
swap. An interrupted run therefore always leaves a loadable FTW, and a rerun finishes the job.
``--out`` writes a fresh, compact FTW dir instead and never touches the original.
"""

from __future__ import annotations

import argparse
import bisect
import errno
import glob
import json
import math
import os
import re
import shutil
import struct
import sys
import tempfile

import torch

from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _is_nvfp4_global, _SHARD_FMT
from freetoken.models.config import VISION_KEY_PREFIXES
from freetoken.distributed.info import set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.engine.engine import _decode_target
from freetoken.layers import set_rope_device
from freetoken.layers.quantization.configs.base import Stored
from freetoken.layers.quantization.names import NameMap
from freetoken.models import create_model
from freetoken.models.register import get_model_spec
from freetoken.utils.progress import byte_bar, count_bar
from freetoken.utils.torch_utils import torch_dtype

_ST_DTYPES = {
    "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16, "F8_E4M3": torch.float8_e4m3fn,
    "F8_E5M2": torch.float8_e5m2, "U8": torch.uint8, "I8": torch.int8, "I32": torch.int32, "I64": torch.int64,
}
_FLOAT_DTYPES = {torch.bfloat16, torch.float16, torch.float32}
_CHUNK = 64 << 20
_SHARD_RE = re.compile(r"^freetoken-(\d{5})\.ftw$")
_PLE_INFIX = ".ple.ple_embedding.ngram_embedding."
_PLE_SHARD_RE = re.compile(r"\.ple\.ple_embedding\.ngram_embedding\.shard_(\d+)\.weight$")
_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"
_PLE_FILE_RE = re.compile(r"^ple-table-\d{5}\.safetensors$")
_PLE_FILE_BYTES = 4 << 30
_BIAS_SUFFIX = ".e_score_correction_bias"
# tensors an old FTW may carry that the text model does not declare; they are kept and never count as errors
_IGNORED_PREFIXES = VISION_KEY_PREFIXES
_VERBOSE = False


def log(msg: str) -> None:
    if _VERBOSE:
        print(msg, flush=True)


def bar(total_bytes: int, desc: str):
    """A byte progress bar in normal mode; verbose mode prints per-step lines instead."""
    return None if _VERBOSE else byte_bar(total_bytes, desc)


def tick(b, n: int) -> None:
    if b is not None:
        b.update(n)


def done(b) -> None:
    if b is not None:
        b.close()


def tensor_bytes(shape, dtype: torch.dtype) -> int:
    return math.prod(shape) * torch.empty((), dtype=dtype).element_size()


# ------------------------------------------------------------------ safetensors slicing
class TensorSource:
    """Single tensors out of an HF repo (byte-range GET) or a local safetensors dir."""

    def __init__(self, repo: str | None, local: str | None, revision: str | None = None):
        assert repo or local, "need --repo or --source to fetch tensors"
        self.repo, self.local, self.revision = repo, local, revision
        self._headers: dict[str, tuple[dict, int]] = {}
        if local:
            files = sorted(os.path.basename(p) for p in glob.glob(os.path.join(local, "*.safetensors")))
            index = os.path.join(local, "model.safetensors.index.json")
        else:
            from huggingface_hub import HfApi, hf_hub_download

            repo_files = HfApi().list_repo_files(repo, revision=revision)
            files = sorted(f for f in repo_files if f.endswith(".safetensors") and "/" not in f)
            index = (hf_hub_download(repo, "model.safetensors.index.json", revision=revision)
                     if "model.safetensors.index.json" in repo_files else None)
        if index and os.path.exists(index):
            with open(index) as f:
                self.weight_map = json.load(f)["weight_map"]
        else:
            self.weight_map = {name: shard for shard in files for name in self._header(shard)[0] if name != "__metadata__"}

    def has(self, name: str) -> bool:
        return name in self.weight_map

    def _range(self, shard: str, start: int, end: int) -> bytes:
        if self.local:
            with open(os.path.join(self.local, shard), "rb") as f:
                f.seek(start)
                return f.read(end - start + 1)
        import requests
        from huggingface_hub import get_token, hf_hub_url

        headers = {"Range": f"bytes={start}-{end}"}
        token = get_token()
        if token:
            headers["Authorization"] = f"Bearer {token}"
        url = hf_hub_url(self.repo, shard, revision=self.revision)
        r = requests.get(url, headers=headers, allow_redirects=True, timeout=300)
        if r.status_code != 206:
            raise RuntimeError(f"range read of {shard} failed: HTTP {r.status_code}")
        return r.content

    def _header(self, shard: str) -> tuple[dict, int]:
        if shard not in self._headers:
            n = struct.unpack("<Q", self._range(shard, 0, 7))[0]
            self._headers[shard] = (json.loads(self._range(shard, 8, 8 + n - 1)), 8 + n)
        return self._headers[shard]

    def meta(self, name: str) -> dict:
        return self._header(self.weight_map[name])[0][name]

    def nbytes(self, name: str) -> int:
        a, b = self.meta(name)["data_offsets"]
        return b - a

    def get(self, name: str) -> torch.Tensor:
        shard = self.weight_map[name]
        header, base = self._header(shard)
        meta = header[name]
        a, b = meta["data_offsets"]
        raw = bytearray(self._range(shard, base + a, base + b - 1))
        return torch.frombuffer(raw, dtype=_ST_DTYPES[meta["dtype"]]).reshape(meta["shape"])


# ------------------------------------------------------------------ what the current build expects
def expected_tensors(ftw_dir: str, resident_experts: bool):
    if try_get_tp_info() is None:
        set_tp_info(0, 1)
    set_rope_device(torch.device("cpu"))
    kw = dict(model_path=ftw_dir, tp_info=try_get_tp_info(), dtype=torch.bfloat16)
    mc0 = EngineConfig(**kw).model_config
    is_moe = bool(getattr(mc0, "is_moe", False) or getattr(mc0, "moe_enabled", False))
    strategy = "offload" if is_moe and not resident_experts else "fused"
    cfg = EngineConfig(**kw, moe_strategy=strategy)
    object.__setattr__(cfg.model_config, "moe_strategy", strategy)
    object.__setattr__(cfg.model_config, "decode_target", _decode_target(cfg))
    with torch.device("meta"), torch_dtype(torch.bfloat16):
        model = create_model(cfg.model_config)
    state = {k: (tuple(v.shape), v.dtype) for k, v in model.state_dict().items()}
    arch = cfg.model_config.architectures[0]
    quant = cfg.model_config.quant
    if quant is None:
        spec = get_model_spec(arch)
        name_map = NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping)
    else:
        name_map = quant.name_map
    return arch, state, name_map, quant


def stored_entry(quant, module: str, role: str) -> Stored:
    """The checkpoint tensor behind ``module``'s ``role`` in the checkpoint's dialect; a plain same-named tensor when the dialect has no say."""
    scheme = quant.scheme_for(module) if quant is not None else None
    if scheme is None:
        return Stored(role)
    return quant.storage(scheme).get(role, Stored(role))


# ------------------------------------------------------------------ repairs
def dsv4_rename(name: str) -> str:
    if name == "head":
        return "model.head.weight"
    if name.endswith(".scale"):
        name = name[: -len(".scale")] + ".weight_scale_inv"
    return "model." + name


def plan(arch: str, entries: list[dict], expected: dict, name_map: NameMap, quant=None):
    """Return (renames, dequants, fetches, drops, leftovers) that turn the FTW dense set into ``expected``."""
    dense = {e["name"]: e for e in entries if e["kind"] == "weight"}
    renames: dict[str, str] = {}
    if arch.startswith("DeepseekV4") and not any(n in expected for n in dense):
        for n in dense:
            new = dsv4_rename(n)
            if new not in expected:
                raise SystemExit(f"DeepSeek-V4 rename has no target for {n!r} -> {new!r}")
            renames[n] = new
    names = {renames.get(n, n): e for n, e in dense.items()}

    dequants: list[tuple[str, dict, dict]] = []
    drops: set[str] = set()
    for n, e in names.items():
        exp = expected.get(n)
        scale = names.get(n[: -len(".weight")] + ".weight_scale") if n.endswith(".weight") else None
        if (exp and exp[1] == torch.bfloat16 and e["dtype"] == "float8_e4m3fn" and tuple(e["shape"]) == exp[0]
                and scale is not None and scale["name"] not in expected and list(scale["shape"]) == [e["shape"][0]]):
            dequants.append((n, e, scale))
            drops.add(scale["name"])

    # (name, checkpoint parts, reciprocal): a fused module maps to several checkpoint tensors, named as the dialect stores the role
    fetches: list[tuple[str, list[str], bool]] = []
    for n in (n for n in expected if n not in names):
        module, _, leaf = n.rpartition(".")
        entry = stored_entry(quant, module, leaf) if module else Stored(leaf)
        parts = [f"{m}.{entry.name}" for m in name_map.to_checkpoint(module)] if module else [n]
        fetches.append((n, parts, entry.reciprocal))

    leftovers = [n for n in names if n not in expected and n not in drops]
    return renames, dequants, fetches, drops, leftovers


def rounded_biases(entries: list[dict], expected: dict, renames: dict[str, str]) -> dict[str, dict]:
    """Router selection biases the FTW stores narrower than the fp32 the model reads; the rounding can change the selected experts."""
    out: dict[str, dict] = {}
    for e in entries:
        name = renames.get(e["name"], e["name"])
        exp = expected.get(name)
        if (e["kind"] == "weight" and name.endswith(_BIAS_SUFFIX) and exp is not None and exp[1] == torch.float32
                and _dtype_of(e["dtype"]) in (torch.bfloat16, torch.float16)):
            out[name] = e
    return out


def resolve_biases(biases: dict[str, dict], source: TensorSource, ftw_dir: str, index: dict) -> tuple[dict[str, torch.Tensor], int, list[str]]:
    """Fetch the fp32 source tensor of each rounded bias; a source that stores it narrow too has nothing to restore."""
    restored: dict[str, torch.Tensor] = {}
    same, errors = 0, []
    for n, e in biases.items():
        # GLM checkpoints keep the bias under the router gate; the readers drop the ``gate.``
        names = [n, n[: -len(_BIAS_SUFFIX)] + ".gate" + _BIAS_SUFFIX]
        src = next((c for c in names if source.has(c)), None)
        if src is None:
            errors.append(f"no source tensor for {n}: tried {names}")
        elif source.meta(src)["dtype"] != "F32":
            same += 1
        else:
            t = source.get(src)
            # the FTW must hold this very tensor, rounded; anything else means a different source checkpoint
            if torch.equal(t.to(_dtype_of(e["dtype"])), read_entry(ftw_dir, index, e)):
                restored[n] = t
            else:
                errors.append(f"{src} in the source does not round to the FTW's {n}; pass the checkpoint this FTW was converted from")
    return restored, same, errors


def resolve_fetches(fetches, source: TensorSource) -> tuple[dict[str, tuple[list[str], bool]], list[str]]:
    """Map each missing tensor to (the source tensors it is built from, reciprocal); a fused scale needs every part."""
    resolved: dict[str, tuple[list[str], bool]] = {}
    errors: list[str] = []
    for n, parts, reciprocal in fetches:
        if all(source.has(p) for p in parts):
            if len(parts) > 1 and not n.endswith(".input_scale"):
                errors.append(f"{n} maps to {len(parts)} source tensors; fusing is not supported here")
            else:
                resolved[n] = (parts, reciprocal)
        else:
            errors.append(f"no source tensor for {n}: missing {[p for p in parts if not source.has(p)]}")
    return resolved, errors


# ------------------------------------------------------------------ vision encoder
_TOWER_SEGMENTS = ("multi_modal_projector", "patch_merge_mlp", "aligner")


def is_vision(name: str) -> bool:
    return name.startswith(VISION_KEY_PREFIXES)


def is_checkpoint_tower_name(name: str) -> bool:
    """A checkpoint tensor of the vision encoder: its first path segment under the wrapper root names the tower, its embedder or its projector."""
    head = name.split(".")
    if head[0] == "model" and len(head) > 1:
        head = head[1:]
    return ("vision" in head[0] or "visual" in head[0] or head[0] in _TOWER_SEGMENTS
            or head[0] in ("aligner", "image_start", "image_end", "image_newline", "image_pad")
            or name.endswith(".ffn.gate.bias_vl"))


def read_tower(source: TensorSource, ftw_dir: str, checkpoint_names: list[str]) -> dict[str, torch.Tensor]:
    """The encoder tensors as the family's encoder-only reader emits them: the tower's checkpoint tensors go into a scratch checkpoint dir next to the FTW's config files, and the reader renames and fuses them the way the converter did."""
    from safetensors.torch import save_file

    from freetoken.models.weight import load_vision_weight

    tmp = tempfile.mkdtemp(prefix="ftw-hotfix-tower-")
    try:
        copy_side_files(ftw_dir, tmp, lambda f: f.endswith(".safetensors") or f == "model.safetensors.index.json")
        b = bar(sum(source.nbytes(n) for n in checkpoint_names), "Fetching the vision encoder")
        tensors: dict[str, torch.Tensor] = {}
        for n in checkpoint_names:
            tensors[n] = source.get(n).contiguous()
            tick(b, tensors[n].numel() * tensors[n].element_size())
        done(b)
        save_file(tensors, os.path.join(tmp, "tower.safetensors"))
        with open(os.path.join(tmp, "model.safetensors.index.json"), "w") as f:
            json.dump({"metadata": {}, "weight_map": {n: "tower.safetensors" for n in checkpoint_names}}, f)
        del tensors
        return {n: t for n, t in load_vision_weight(tmp, torch.device("cpu")) if is_vision(n)}
    finally:
        shutil.rmtree(tmp, ignore_errors=True)


def dequantize_rows(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
    return (weight.float() * scale.float()[:, None]).to(torch.bfloat16)


def validate_ftw(ftw_dir: str, index: dict, expected: dict, renames: dict[str, str], dequant_names: set[str]) -> list[str]:
    """Structural checks of the index and shard files, plus shape/dtype agreement with the model."""
    problems: list[str] = []
    pos = 0
    for sh in sorted(index["shards"], key=lambda s: s["global_off"]):
        if sh["global_off"] != pos:
            problems.append(f"shard {sh['file']} starts at {sh['global_off']}, expected {pos}")
        path = os.path.join(ftw_dir, sh["file"])
        if not os.path.exists(path):
            problems.append(f"shard file {sh['file']} is missing")
        elif os.path.getsize(path) != sh["nbytes"]:
            problems.append(f"shard {sh['file']} is {os.path.getsize(path)} B, index says {sh['nbytes']}")
        pos = sh["global_off"] + sh["nbytes"]
    if pos != index["total_bytes"]:
        problems.append(f"shards end at {pos}, index total_bytes is {index['total_bytes']}")
    seen: set[str] = set()
    for e in index["tensors"]:
        if e["name"] in seen:
            problems.append(f"duplicate entry {e['name']}")
        seen.add(e["name"])
        try:
            dt = _dtype_of(e["dtype"])
        except Exception:
            problems.append(f"{e['name']}: unknown dtype {e['dtype']!r}")
            continue
        if tensor_bytes(e["shape"], dt) != e["nbytes"]:
            problems.append(f"{e['name']}: {e['shape']} {e['dtype']} is {tensor_bytes(e['shape'], dt)} B, index says {e['nbytes']}")
        if e["global_off"] < 0 or e["global_off"] % ALIGN:
            problems.append(f"{e['name']}: offset {e['global_off']} is not {ALIGN}-aligned")
        if e["global_off"] + e["nbytes"] > pos:
            problems.append(f"{e['name']}: extends past the last shard")
        if e["kind"] != "weight":
            continue
        name = renames.get(e["name"], e["name"])
        exp = expected.get(name)
        if exp is None:
            continue
        if name in dequant_names:  # shape already checked by plan(); the dtype is what the repair changes
            continue
        if tuple(e["shape"]) != exp[0]:
            problems.append(f"{name}: FTW shape {e['shape']} != model shape {list(exp[0])}")
        # the loader casts between float types; anything else has to match exactly
        if dt != exp[1] and not (dt in _FLOAT_DTYPES and exp[1] in _FLOAT_DTYPES):
            problems.append(f"{name}: FTW dtype {e['dtype']} is not loadable as {exp[1]}")
    fp16_globals = [e["name"] for e in index["tensors"] if e["dtype"] == "float16" and _is_nvfp4_global(e["name"])]
    if fp16_globals:
        problems.append(f"{len(fp16_globals)} NVFP4 global scale entries are fp16 (e.g. {fp16_globals[0]}) but FreeToken reads them as fp32; "
                        "this script cannot repair that, reconvert the FTW with `ft checkpoint`")
    blocks = sorted((e["global_off"], e["global_off"] + _align_up(e["nbytes"]), e["name"]) for e in index["tensors"])
    for (_, a1, an), (b0, _, bn) in zip(blocks, blocks[1:]):
        if b0 < a1:
            problems.append(f"{bn} overlaps {an}")
    return problems


# ------------------------------------------------------------------ FTW I/O
class ShardWriter:
    """Streams entries into ``freetoken-NNNNN.ftw`` shards, from offset 0 (new dir) or from the old end (append)."""

    def __init__(self, out_dir: str, shard_limit: int, *, first_shard: int = 0, global_off: int = 0):
        self.out_dir, self.shard_limit = out_dir, shard_limit
        self.global_off = global_off
        self.first_shard = first_shard
        self.shard_idx = first_shard - 1
        self._f = None
        self._start = self._cur = 0
        self.shards: list[dict] = []
        self.entries: list[dict] = []

    def _path(self, idx: int) -> str:
        return os.path.join(self.out_dir, _SHARD_FMT.format(idx))

    def _finish_shard(self) -> None:
        self.shards.append({"file": _SHARD_FMT.format(self.shard_idx), "global_off": self._start, "nbytes": self._cur})
        self._f.flush()
        os.fsync(self._f.fileno())
        self._f.close()
        self._f = None

    def _roll(self) -> None:
        if self._f is not None:
            self._finish_shard()
        self.shard_idx += 1
        self._start, self._cur = self.global_off, 0
        self._f = open(self._path(self.shard_idx), "wb")

    def _write(self, data) -> None:
        off, n = 0, len(data)
        while off < n:
            if self._f is None or self._cur == self.shard_limit:
                self._roll()
            take = min(n - off, self.shard_limit - self._cur)
            self._f.write(data[off:off + take])
            off += take
            self._cur += take
            self.global_off += take

    def add(self, entry: dict, chunks) -> None:
        nbytes = entry["nbytes"]
        # a tensor that fits a shard never straddles two: roll early like FTWWriter does
        if self._f is None or (nbytes <= self.shard_limit and self._cur + nbytes > self.shard_limit):
            self._roll()
        assert self.global_off % ALIGN == 0
        self.entries.append({**entry, "global_off": self.global_off})
        written = 0
        for c in chunks:
            self._write(c)
            written += len(c)
        assert written == nbytes, (entry["name"], written, nbytes)
        pad = _align_up(self.global_off) - self.global_off
        if pad:
            self._write(bytes(pad))

    def add_tensor(self, name: str, t: torch.Tensor) -> None:
        t = t.detach().cpu().contiguous()
        raw = t.reshape(-1).view(torch.uint8).numpy().tobytes()
        self.add({"name": name, "kind": "weight", "dtype": _dtype_str(t.dtype), "shape": list(t.shape), "nbytes": len(raw)}, [raw])

    def close(self) -> list[dict]:
        if self._f is not None:
            self._finish_shard()
        return self.shards

    def abort(self) -> None:
        if self._f is not None:
            self._f.close()
            self._f = None
        for idx in range(self.first_shard, self.shard_idx + 1):
            try:
                os.remove(self._path(idx))
            except FileNotFoundError:
                pass


def entry_chunks(ftw_dir: str, index: dict, e: dict):
    """The bytes of an existing entry, read from its shards in pieces of at most _CHUNK."""
    pos, remaining = e["global_off"], e["nbytes"]
    for sh in sorted(index["shards"], key=lambda s: s["global_off"]):
        s0, s1 = sh["global_off"], sh["global_off"] + sh["nbytes"]
        if remaining <= 0 or pos >= s1:
            continue
        with open(os.path.join(ftw_dir, sh["file"]), "rb") as f:
            f.seek(pos - s0)
            take = min(s1 - pos, remaining)
            while take > 0:
                buf = f.read(min(take, _CHUNK))
                if not buf:
                    raise ValueError(f"short read in {sh['file']} for {e['name']}")
                yield buf
                take -= len(buf)
                pos += len(buf)
                remaining -= len(buf)
    assert remaining == 0, (e["name"], remaining)


def read_entry(ftw_dir: str, index: dict, e: dict) -> torch.Tensor:
    buf = bytearray(b"".join(entry_chunks(ftw_dir, index, e)))
    return torch.frombuffer(buf, dtype=_dtype_of(e["dtype"])).reshape(e["shape"])


def fsync_path(path: str) -> None:
    fd = os.open(path, os.O_RDONLY)
    try:
        os.fsync(fd)
    except OSError as e:
        # a filesystem may not support fsync on a directory; any other error is a real persistence failure
        if not (os.path.isdir(path) and e.errno in (errno.EINVAL, errno.ENOTSUP, errno.EOPNOTSUPP)):
            raise
    finally:
        os.close(fd)


def write_index(ftw_dir: str, index: dict) -> None:
    # data of every file the new index points at is synced by its writer; sync the index, then the rename
    tmp = os.path.join(ftw_dir, INDEX_NAME + ".tmp")
    with open(tmp, "w") as f:
        json.dump(index, f)
        f.flush()
        os.fsync(f.fileno())
    os.replace(tmp, os.path.join(ftw_dir, INDEX_NAME))
    fsync_path(ftw_dir)


def shard_numbers(ftw_dir: str, index: dict) -> tuple[int, list[str]]:
    """The next free shard number, and shard files on disk that the index does not reference."""
    used = {sh["file"] for sh in index["shards"]}
    on_disk = [f for f in os.listdir(ftw_dir) if _SHARD_RE.match(f)]
    nums = [int(_SHARD_RE.match(f).group(1)) for f in set(on_disk) | used if _SHARD_RE.match(f)]
    return (max(nums) + 1 if nums else 0), sorted(f for f in on_disk if f not in used)


# ------------------------------------------------------------------ compaction
def live_ranges(tensors: list[dict], s0: int, s1: int) -> list[tuple[int, int]]:
    """Referenced byte ranges of shard [s0, s1), merged and sorted; every entry owns its ALIGN-padded block."""
    rs = []
    for e in tensors:
        a, b = e["global_off"], e["global_off"] + _align_up(e["nbytes"])
        a, b = max(a, s0), min(b, s1)
        if a < b:
            rs.append((a, b))
    rs.sort()
    merged: list[tuple[int, int]] = []
    for a, b in rs:
        if merged and a <= merged[-1][1]:
            merged[-1] = (merged[-1][0], max(merged[-1][1], b))
        else:
            merged.append((a, b))
    return merged


def dirty_shards(index: dict) -> list[tuple[dict, list[tuple[int, int]], int]]:
    """Shards holding bytes no entry references, with their live ranges and live byte count."""
    out = []
    for sh in sorted(index["shards"], key=lambda s: s["global_off"]):
        s0, s1 = sh["global_off"], sh["global_off"] + sh["nbytes"]
        live = live_ranges(index["tensors"], s0, s1)
        live_bytes = sum(b - a for a, b in live)
        if live_bytes < sh["nbytes"]:
            out.append((sh, live, live_bytes))
    return out


def compact_in_place(work: str, index: dict, next_num: int) -> tuple[dict, int]:
    """Rewrite each dirty shard without its dead bytes; one atomic index swap per shard."""
    dirty = dirty_shards(index)
    b = bar(sum(lb for _, _, lb in dirty), "Compacting shards") if dirty else None
    n = 0
    while dirty:
        sh, live, live_bytes = dirty[0]
        s0, s1 = sh["global_off"], sh["global_off"] + sh["nbytes"]
        delta = sh["nbytes"] - live_bytes
        old_path = os.path.join(work, sh["file"])
        new_file = None
        if live_bytes:
            new_file = _SHARD_FMT.format(next_num)
            next_num += 1
            log(f"  compact {sh['file']} -> {new_file}: keep {live_bytes} B, drop {delta} B")
            with open(old_path, "rb") as src, open(os.path.join(work, new_file), "wb") as dst:
                for a, b1 in live:
                    src.seek(a - s0)
                    left = b1 - a
                    while left > 0:
                        buf = src.read(min(left, _CHUNK))
                        if not buf:
                            raise ValueError(f"short read in {sh['file']}")
                        dst.write(buf)
                        left -= len(buf)
                        tick(b, len(buf))
                dst.flush()
                os.fsync(dst.fileno())
        else:
            log(f"  drop empty shard {sh['file']}")
        starts = [a for a, _ in live]
        prefix = [0]
        for a, b1 in live:
            prefix.append(prefix[-1] + (b1 - a))

        def remap(go: int) -> int:
            i = bisect.bisect_right(starts, go) - 1
            if i < 0 or go >= live[i][1]:
                # only a zero-byte entry can sit in a dead gap: park it at the next live block
                return s0 + prefix[i + 1]
            return s0 + prefix[i] + (go - starts[i])

        tensors = []
        for e in index["tensors"]:
            go = e["global_off"]
            if go >= s1:
                e = {**e, "global_off": go - delta}
            elif go >= s0:
                e = {**e, "global_off": remap(go)}
            tensors.append(e)
        shards = []
        for x in sorted(index["shards"], key=lambda s: s["global_off"]):
            if x["file"] == sh["file"]:
                if new_file:
                    shards.append({"file": new_file, "global_off": s0, "nbytes": live_bytes})
            elif x["global_off"] > s0:
                shards.append({**x, "global_off": x["global_off"] - delta})
            else:
                shards.append(x)
        index = {**index, "tensors": tensors, "shards": shards, "total_bytes": index["total_bytes"] - delta}
        write_index(work, index)
        os.remove(old_path)
        n += 1
        dirty = dirty_shards(index)
    done(b)
    return index, n


# ------------------------------------------------------------------ PLE table
class PleSpec:
    """What the loader demands of the table: split_ngram_parts fp8 shards of [rows, ngram_head_dim] and one scalar scale."""

    def __init__(self, ftw_dir: str):
        with open(os.path.join(ftw_dir, "config.json")) as f:
            cfg = json.load(f)
        t = cfg.get("text_config") or cfg
        self.n_shards = int(t["split_ngram_parts"])
        self.head_dim = int(t["ple_embed_dim"]) // ((int(t["ngram_size"]) - 1) * int(t["heads_per_ngram"]))


def _ple_collect(items, bad: list[str]) -> tuple[dict[int, tuple[tuple, int]], int | None]:
    """Group (name, dtype, shape, nbytes) rows into shard index -> (shape, nbytes) and the scale's element count."""
    shards: dict[int, tuple[tuple, int]] = {}
    scale_numel = None
    for name, dtype, shape, nbytes in items:
        if name.endswith(_PLE_SCALE_SUFFIX):
            scale_numel = math.prod(shape)
            continue
        mt = _PLE_SHARD_RE.search(name)
        if mt is None:
            continue
        idx = int(mt.group(1))
        if dtype != "F8_E4M3":
            bad.append(f"shard {idx} has dtype {dtype}")
        if idx in shards:
            bad.append(f"duplicate shard {idx}")
        shards[idx] = (tuple(shape), nbytes)
    return shards, scale_numel


def classify_ple(spec: PleSpec, shards: dict[int, tuple[tuple, int]], scale_numel: int | None, bad: list[str]) -> tuple[str, str]:
    """('complete' | 'missing' | 'partial', detail) by the loader's rules."""
    if not shards and scale_numel is None and not bad:
        return "missing", ""
    for idx, (shape, nbytes) in sorted(shards.items()):
        if len(shape) != 2 or shape[1] != spec.head_dim:
            bad.append(f"shard {idx} is {list(shape)}, expected [rows, {spec.head_dim}]")
        elif nbytes != shape[0] * shape[1]:
            bad.append(f"shard {idx} holds {nbytes} B for shape {list(shape)}")
    if len({shape for shape, _ in shards.values()}) > 1:
        bad.append("shards differ in shape")
    if scale_numel is not None and scale_numel != 1:
        bad.append(f"weight_scale has {scale_numel} elements")
    if bad:
        return "partial", "; ".join(bad[:3])
    if scale_numel is None or sorted(shards) != list(range(spec.n_shards)):
        return "partial", f"{len(shards)}/{spec.n_shards} shards" + ("" if scale_numel is not None else ", no weight_scale")
    return "complete", ""


def ple_source_status(source: TensorSource, spec: PleSpec) -> tuple[str, str]:
    bad: list[str] = []
    names = [k for k in source.weight_map if _PLE_INFIX in k]
    shards, scale = _ple_collect(((k, source.meta(k)["dtype"], source.meta(k)["shape"], source.nbytes(k)) for k in names), bad)
    return classify_ple(spec, shards, scale, bad)


def ple_table_files(ftw_dir: str) -> list[str]:
    """The files the loader reads the table from: the safetensors index's mapping when there is one, else every *.safetensors."""
    index = os.path.join(ftw_dir, "model.safetensors.index.json")
    if not os.path.exists(index):
        return sorted(glob.glob(os.path.join(ftw_dir, "*.safetensors")))
    with open(index) as f:
        weight_map = json.load(f)["weight_map"]
    return sorted({os.path.join(ftw_dir, sh) for n, sh in weight_map.items() if _PLE_INFIX in n})


def ple_table_status(ftw_dir: str, spec: PleSpec) -> tuple[str, str, list[str]]:
    """The dir's PLE table as the loader would see it: (status, detail, files holding PLE tensors)."""
    bad: list[str] = []
    items: list[tuple] = []
    files: list[str] = []
    for path in ple_table_files(ftw_dir):
        name = os.path.basename(path)
        if not os.path.exists(path):
            bad.append(f"model.safetensors.index.json maps PLE tensors to missing {name}")
            continue
        try:
            with open(path, "rb") as f:
                n = struct.unpack("<Q", f.read(8))[0]
                header, base = json.loads(f.read(n)), 8 + n
        except (struct.error, ValueError):
            bad.append(f"{name} has an unreadable header")
            files.append(name)
            continue
        size = os.path.getsize(path)
        holds = False
        for k, m in header.items():
            if k == "__metadata__" or _PLE_INFIX not in k:
                continue
            holds = True
            a, b = m["data_offsets"]
            if base + b > size:
                bad.append(f"{name} is truncated")
            items.append((k, m["dtype"], m["shape"], b - a))
        if holds:
            files.append(name)
    shards, scale = _ple_collect(items, bad)
    status, detail = classify_ple(spec, shards, scale, bad)
    return status, detail, files


def check_ftw(ftw_dir: str, index: dict, expected: dict, renames: dict[str, str], dequant_names: set[str], ple: PleSpec | None) -> dict:
    """Everything the tool verifies, on the input before planning and on the result after writing."""
    have = {renames.get(e["name"], e["name"]) for e in index["tensors"] if e["kind"] == "weight"}
    return {
        "problems": validate_ftw(ftw_dir, index, expected, renames, dequant_names),
        "missing": [n for n in expected if n not in have],
        "extra": [n for n in have if n not in expected and not n.startswith(_IGNORED_PREFIXES)],
        "dead": dirty_shards(index),
        "ple": ple_table_status(ftw_dir, ple) if ple else ("n/a", "", []),
    }


def ple_shards(out_dir: str, source: TensorSource) -> list[str]:
    """Write the PLE n-gram table tensors, and nothing else, into ``ple-table-*.safetensors`` files in ``out_dir``."""
    from safetensors.torch import save_file

    todo = sorted(n for n in source.weight_map if _PLE_INFIX in n)
    if not todo:
        raise SystemExit("ERROR: the source checkpoint has no PLE table tensors")
    for stale in glob.glob(os.path.join(out_dir, "ple-table-*.safetensors.tmp")):
        os.remove(stale)
    names: list[str] = []
    batch: dict[str, torch.Tensor] = {}
    size = 0
    b = bar(sum(source.nbytes(n) for n in todo), "Writing PLE table")

    def flush():
        nonlocal batch, size
        if not batch:
            return
        name = f"ple-table-{len(names):05d}.safetensors"
        log(f"  write {name}: {len(batch)} tensors, {size / 2**30:.2f} GiB")
        save_file(batch, os.path.join(out_dir, name + ".tmp"))
        names.append(name)
        batch, size = {}, 0

    try:
        for n in todo:
            t = source.get(n)
            batch[n] = t
            size += t.numel() * t.element_size()
            tick(b, t.numel() * t.element_size())
            if size >= _PLE_FILE_BYTES:
                flush()
        flush()
    except BaseException:
        for name in names:
            os.remove(os.path.join(out_dir, name + ".tmp"))
        raise
    done(b)
    # every file is complete and synced before any gets its final name, so a scan never sees a half table
    for name in names:
        fsync_path(os.path.join(out_dir, name + ".tmp"))
        os.replace(os.path.join(out_dir, name + ".tmp"), os.path.join(out_dir, name))
    fsync_path(out_dir)
    return names


def side_files(src: str, skip) -> list[str]:
    """Everything in the FTW dir except the shards and the index: configs, tokenizer, PLE tables, nested dirs."""
    return [f for f in os.listdir(src) if not (f.endswith((".ftw", ".bak", ".tmp")) or f == INDEX_NAME or skip(f))]


def copy_side_files(src: str, dst: str, skip) -> None:
    os.makedirs(dst, exist_ok=True)
    for f in side_files(src, skip):
        s = os.path.join(src, f)
        if os.path.isdir(s):
            shutil.copytree(s, os.path.join(dst, f), dirs_exist_ok=True)
        else:
            shutil.copy2(s, os.path.join(dst, f))


def side_files_bytes(src: str, skip) -> int:
    total = 0
    for f in side_files(src, skip):
        p = os.path.join(src, f)
        total += sum(os.path.getsize(os.path.join(r, x)) for r, _, fs in os.walk(p) for x in fs) if os.path.isdir(p) else os.path.getsize(p)
    return total


# ------------------------------------------------------------------ main
def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--ftw", required=True, help="FTW checkpoint dir to repair")
    p.add_argument("--out", help="write a fresh, compact FTW dir here (new or empty) instead of patching in place")
    p.add_argument("--repo", help="HF repo id of the source checkpoint (tensors are read by byte range)")
    p.add_argument("--revision", help="HF revision (branch, tag or commit) of --repo")
    p.add_argument("--source", help="local HF safetensors dir of the source checkpoint (instead of --repo)")
    p.add_argument("--dry-run", action="store_true")
    p.add_argument("-v", "--verbose", action="store_true", help="print every step")
    ns = p.parse_args(argv)
    global _VERBOSE
    _VERBOSE = ns.verbose

    if not os.path.isfile(os.path.join(ns.ftw, INDEX_NAME)):
        print(f"ERROR: {ns.ftw} has no {INDEX_NAME}; not an FTW checkpoint dir", file=sys.stderr)
        return 2
    with open(os.path.join(ns.ftw, INDEX_NAME)) as f:
        index = json.load(f)
    if index.get("format") != FORMAT_TAG:
        print(f"ERROR: {ns.ftw} is not an FTW checkpoint (format {index.get('format')!r})", file=sys.stderr)
        return 2
    if ns.out:
        out, ftw = os.path.abspath(ns.out), os.path.abspath(ns.ftw)
        if out == ftw or os.path.commonpath([out, ftw]) == ftw:
            print("ERROR: --out must be outside --ftw (omit --out to patch in place)", file=sys.stderr)
            return 2
        if os.path.exists(out) and not os.path.isdir(out):
            print(f"ERROR: --out {ns.out} exists and is not a directory", file=sys.stderr)
            return 2
        if os.path.isdir(out) and os.listdir(out):
            print(f"ERROR: --out {ns.out} exists and is not empty", file=sys.stderr)
            return 2
    if ns.revision and not ns.repo:
        print("  note: --revision only applies to --repo; ignored", file=sys.stderr)
    resident_experts = any(e["kind"] == "weight" and ".experts." in e["name"] for e in index["tensors"])
    log(f"reading {os.path.join(ns.ftw, INDEX_NAME)}: {len(index['tensors'])} entries, {len(index['shards'])} shards, {index['total_bytes'] / 2**30:.2f} GiB")
    log("building the current model on the meta device from the FTW's config.json" + (" (resident experts)" if resident_experts else ""))
    arch, expected, name_map, quant = expected_tensors(ns.ftw, resident_experts)
    log(f"{arch}: model declares {len(expected)} dense tensors")
    source = TensorSource(ns.repo, ns.source, ns.revision) if (ns.repo or ns.source) else None
    renames, dequants, fetches, drops, leftovers = plan(arch, index["tensors"], expected, name_map, quant)
    # only the old GLM-5.2 reader rounded the bias; other families store it as the checkpoint does
    biases = rounded_biases(index["tensors"], expected, renames) if arch == "GlmMoeDsaForCausalLM" else {}
    # the encoder comes from the family reader, not from by-name fetches
    vision_missing = sorted(n for n, *_ in fetches if is_vision(n))
    fetches = [f for f in fetches if not is_vision(f[0])]
    dequant_names = {n for n, _, _ in dequants}
    ple = PleSpec(ns.ftw) if arch.startswith("Qwen4Exp") else None
    chk = check_ftw(ns.ftw, index, expected, renames, dequant_names, ple)
    problems, dirty = chk["problems"], chk["dead"]
    ple_status, ple_detail, ple_files = chk["ple"]
    need_ple = ple_status in ("missing", "partial")
    next_num, orphans = shard_numbers(ns.ftw, index)
    bias_vals: dict[str, torch.Tensor] = {}
    bias_same, bias_errors = 0, []
    if biases and source is not None and not problems:
        bias_vals, bias_same, bias_errors = resolve_biases(biases, source, ns.ftw, index)

    print(f"{arch}: {len(expected)} expected dense tensors, FTW has {sum(e['kind'] == 'weight' for e in index['tensors'])}")
    print(f"  renames {len(renames)}  dequantize {len(dequants)}  fetch {len(fetches)}  drop {len(drops)}  leftover {len(leftovers)}"
          + (f"  vision encoder: {len(vision_missing)} tensors" if vision_missing else "")
          + (f"  PLE table: {ple_status}" + (f" ({ple_detail})" if ple_detail else "") if need_ple else "")
          + (f"  dead bytes in {len(dirty)} shard(s)" if dirty else ""))
    for n, parts, _ in fetches[:8]:
        print(f"    fetch {n} <- {parts}")
    if len(fetches) > 8:
        print(f"    ... {len(fetches) - 8} more")
    for old, new in list(renames.items())[:5]:
        log(f"    rename {old} -> {new}")
    if len(renames) > 5:
        log(f"    ... {len(renames) - 5} more renames")
    for n, e, scale_e in dequants:
        log(f"    dequantize {n} (drop {scale_e['name']})")
    for n in leftovers[:8]:
        print(f"    leftover (not declared by the model): {n}")
    if orphans:
        removing = not ns.out and bool(renames or dequants or fetches or vision_missing or bias_vals or drops or need_ple or dirty)
        print(f"  note: {len(orphans)} shard file(s) not referenced by the index, left by an interrupted run"
              + (", will be removed: " if removing else ", left as is: ") + str(orphans[:3]))

    bad = False
    for msg in problems[:10]:
        print(f"ERROR: {msg}", file=sys.stderr)
        bad = True
    if len(problems) > 10:
        print(f"ERROR: ... {len(problems) - 10} more", file=sys.stderr)
    if (fetches or need_ple or vision_missing) and source is None:
        print("ERROR: tensors must be fetched but neither --repo nor --source was given", file=sys.stderr)
        return 2
    tower_names = [n for n in source.weight_map if is_checkpoint_tower_name(n)] if vision_missing else []
    if vision_missing and not tower_names:
        print("ERROR: the source checkpoint has no vision encoder tensors", file=sys.stderr)
        bad = True
    elif vision_missing:
        print(f"  vision encoder: {len(vision_missing)} tensors from the family reader over {len(tower_names)} checkpoint tensors "
              f"({sum(source.nbytes(n) for n in tower_names) / 2**30:.2f} GiB)")
    fetch_srcs: dict[str, tuple[list[str], bool]] = {}
    if fetches:
        fetch_srcs, errors = resolve_fetches(fetches, source)
        for msg in errors:
            print(f"ERROR: {msg}", file=sys.stderr)
            bad = True
    if biases and source is None:
        print(f"  note: {len(biases)} router selection biases (e.g. {next(iter(biases))}) are stored narrower than the fp32 the model reads; "
              "rerun with --repo or --source to restore them")
    for msg in bias_errors:
        print(f"ERROR: {msg}", file=sys.stderr)
        bad = True
    if bias_vals:
        print(f"  restore {len(bias_vals)} router selection biases in fp32 from the source")
    if bias_same:
        print(f"  {bias_same} router selection biases are narrow in the source as well; nothing to restore")
    # the converter transforms most tensors on the way in (norm offsets, fusion, packing); only the
    # activation scale scalars are stored as the checkpoint has them, so only they can be fetched raw
    unfetchable = [n for n, *_ in fetches if not n.endswith(".input_scale")]
    if unfetchable:
        print(f"ERROR: {len(unfetchable)} missing tensor(s) cannot be fetched raw (first: {unfetchable[0]}); reconvert the checkpoint", file=sys.stderr)
        bad = True
    real_leftovers = [n for n in leftovers if not n.startswith(_IGNORED_PREFIXES)]
    if real_leftovers:
        print(f"ERROR: the FTW holds {len(real_leftovers)} tensors the current model does not declare (first: {real_leftovers[0]}); this layout is not supported", file=sys.stderr)
        bad = True
    foreign_ple = [f for f in ple_files if not _PLE_FILE_RE.match(f)]
    if ple_status == "partial" and foreign_ple:
        print(f"ERROR: the PLE table in {ns.ftw} is incomplete ({ple_detail}) and lives in files this tool did not write: {foreign_ple[:3]}; remove them first", file=sys.stderr)
        bad = True
    if need_ple and os.path.exists(os.path.join(ns.ftw, "model.safetensors.index.json")):
        print(f"ERROR: {ns.ftw}/model.safetensors.index.json makes the loader read the PLE table through its mapping, which cannot list the files this tool writes; remove it (an FTW dir does not need it) and rerun", file=sys.stderr)
        bad = True
    if need_ple and source is not None:
        src_status, src_detail = ple_source_status(source, ple)
        if src_status != "complete":
            print(f"ERROR: the PLE table in the source is {src_status}" + (f" ({src_detail})" if src_detail else ""), file=sys.stderr)
            bad = True
    if bad:
        return 2

    tower: dict[str, torch.Tensor] = {}
    if vision_missing and not ns.dry_run:
        tower = read_tower(source, ns.ftw, tower_names)
        unproduced = [n for n in vision_missing if n not in tower]
        wrong = [n for n in vision_missing if n in tower and tuple(tower[n].shape) != expected[n][0]]
        if unproduced or wrong:
            for n in unproduced[:5]:
                print(f"ERROR: the family reader did not produce {n} from the source's encoder tensors", file=sys.stderr)
            for n in wrong[:5]:
                print(f"ERROR: {n}: reader shape {list(tower[n].shape)} != model shape {list(expected[n][0])}", file=sys.stderr)
            return 2

    replaced = dequant_names | set(fetch_srcs) | set(vision_missing) | set(bias_vals)
    keep: list[tuple[dict, str]] = []
    for e in index["tensors"]:
        name = renames.get(e["name"], e["name"]) if e["kind"] == "weight" else e["name"]
        if e["kind"] == "weight" and (name in drops or name in replaced):
            continue
        keep.append((e, name))
    kept_names = {e["name"] for e, _ in keep}
    skip_ple = (lambda f: bool(_PLE_FILE_RE.match(f))) if need_ple else (lambda f: False)

    # disk: the new entries and the PLE table, plus either one compacted shard next to its original
    # (in place) or the whole compact copy (--out)
    new_bytes = sum(_align_up(tensor_bytes(tower[n].shape, tower[n].dtype) if n in tower else tensor_bytes(*expected[n])) for n in replaced)
    table_bytes = sum(source.nbytes(n) for n in source.weight_map if _PLE_INFIX in n) if need_ple else 0
    work = ns.out or ns.ftw
    if ns.out:
        need = sum(_align_up(e["nbytes"]) for e, _ in keep) + new_bytes + table_bytes + side_files_bytes(ns.ftw, skip_ple)
    else:
        after = {**index, "tensors": [e for e in index["tensors"] if e["name"] in kept_names]}
        need = new_bytes + table_bytes + max([lb for _, _, lb in dirty_shards(after)] + [0])
    probe = os.path.abspath(work)
    while not os.path.isdir(probe):
        probe = os.path.dirname(probe)
    free = shutil.disk_usage(probe).free
    print(f"  disk: about {need / 2**30:.1f} GiB needed under {work} ({free / 2**30:.1f} GiB free)")
    if ns.dry_run:
        return 0
    if not (renames or dequants or fetch_srcs or vision_missing or bias_vals or drops or need_ple or dirty or ns.out):
        print("nothing to do; the FTW loads as is")
        return 0
    if free < need + (1 << 30):
        print(f"ERROR: not enough free space under {work}", file=sys.stderr)
        return 2

    def write_new(w: ShardWriter) -> None:
        b = bar(sum(e["nbytes"] for _, e, _ in dequants), "Dequantizing") if dequants else None
        for n, e, scale_e in dequants:
            log(f"  dequantize {n}: fp8 x {scale_e['name']} -> bf16 {e['shape']}")
            w.add_tensor(n, dequantize_rows(read_entry(ns.ftw, index, e), read_entry(ns.ftw, index, scale_e)))
            tick(b, e["nbytes"])
        done(b)
        items = list(fetch_srcs.items())
        for n in (vision_missing if _VERBOSE or not vision_missing else count_bar(vision_missing, "Writing the vision encoder")):
            log(f"  encoder {n} {list(tower[n].shape)}")
            w.add_tensor(n, tower[n])
        for n, (srcs, reciprocal) in (items if _VERBOSE or not items else count_bar(items, "Fetching tensors")):
            log(f"  fetch {n} <- {', '.join(srcs)}" + (" (reciprocal)" if reciprocal else ""))
            vals = [source.get(c).reshape(()).float() for c in srcs]
            # llm-compressor stores the quant-side global; the layer wants the dequant-side scale, as the reader loads it
            if reciprocal:
                vals = [1.0 / v for v in vals]
            # a fused projection shares one activation scale; the reader takes the max over its parts
            w.add_tensor(n, torch.stack(vals).max().reshape(()))
        for n, t in bias_vals.items():
            log(f"  restore {n} in fp32")
            w.add_tensor(n, t)

    hotfix = {"from": os.path.abspath(ns.ftw), "renamed": len(renames), "dequantized": len(dequants),
              "fetched": len(fetch_srcs) + len(vision_missing), "restored": len(bias_vals), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}
    compacted = 0
    if ns.out:
        log(f"--out: writing a fresh FTW -> {work}")
        try:
            copy_side_files(ns.ftw, work, skip_ple)
            w = ShardWriter(work, index["shard_limit"])
            b = bar(sum(e["nbytes"] for e, _ in keep), "Rewriting shards")
            for e, name in keep:
                log(f"  copy {e['name']}" + (f" -> {name}" if name != e["name"] else "") + f"  {e['nbytes']} B")
                w.add({**e, "name": name}, entry_chunks(ns.ftw, index, e))
                tick(b, e["nbytes"])
            done(b)
            write_new(w)
            shards = w.close()
            side = [f for f in index.get("side_files", []) if not skip_ple(f)]
            if need_ple:
                got = ple_shards(work, source)
                side += got
                print(f"  PLE table written: {len(got)} files")
            new_index = {**index, "tensors": w.entries, "shards": shards, "total_bytes": w.global_off,
                         "counts": {**index.get("counts", {}), "weight": sum(e["kind"] == "weight" for e in w.entries)},
                         "side_files": sorted(side), "hotfix": hotfix}
            write_index(work, new_index)
        except BaseException:
            # the dir was new or empty, so nothing of the user's is in it
            shutil.rmtree(work, ignore_errors=True)
            raise
        added = len(w.entries) - len(keep)
    else:
        for f in orphans:
            os.remove(os.path.join(work, f))
        new_entries: list[dict] = []
        new_shards: list[dict] = []
        if dequants or fetch_srcs or vision_missing or bias_vals:
            log(f"in place: appending shard {_SHARD_FMT.format(next_num)} -> {work}")
            w = ShardWriter(work, index["shard_limit"], first_shard=next_num, global_off=index["total_bytes"])
            try:
                write_new(w)
            except BaseException:
                w.abort()
                raise
            new_shards = w.close()
            new_entries = w.entries
            next_num = w.shard_idx + 1
        tensors = [{**e, "name": name} for e, name in keep] + new_entries
        new_index = {**index, "tensors": tensors, "shards": index["shards"] + new_shards,
                     "total_bytes": index["total_bytes"] + sum(sh["nbytes"] for sh in new_shards),
                     "counts": {**index.get("counts", {}), "weight": sum(e["kind"] == "weight" for e in tensors)},
                     "hotfix": hotfix}
        if renames or new_entries or drops:
            # rollback stays possible only while no shard gets replaced: keep the old index then
            bak = os.path.join(work, INDEX_NAME + ".bak")
            if not drops and not dirty and not bias_vals and not os.path.exists(bak):
                shutil.copy2(os.path.join(work, INDEX_NAME), bak)
            write_index(work, new_index)
            log(f"index written: {os.path.join(work, INDEX_NAME)}")
        if need_ple:
            for f in ple_files:
                os.remove(os.path.join(work, f))
            got = ple_shards(work, source)
            side = {f for f in new_index.get("side_files", []) if not _PLE_FILE_RE.match(f)} | set(got)
            new_index = {**new_index, "side_files": sorted(side)}
            write_index(work, new_index)
            print(f"  PLE table written: {len(got)} files")
        new_index, compacted = compact_in_place(work, new_index, next_num)
        if compacted:
            new_index = {**new_index, "hotfix": {**hotfix, "compacted": compacted}}
            write_index(work, new_index)
            # the old shards are gone, so an old index can no longer roll anything back
            bak = os.path.join(work, INDEX_NAME + ".bak")
            if os.path.exists(bak):
                os.remove(bak)
        added = len(new_entries)

    chk = check_ftw(work, new_index, expected, {}, set(), ple)
    ple_status, ple_detail, _ = chk["ple"]
    print(f"wrote {work}: {len(new_index['tensors'])} entries, +{added} new, {len(new_index['shards'])} shard(s)"
          + (f", {compacted} compacted" if compacted else "") + f", total {new_index['total_bytes'] / 2**30:.2f} GiB")
    print(f"  check: missing {len(chk['missing'])}  extra {len(chk['extra'])}  problems {len(chk['problems'])}  "
          f"dead bytes in {len(chk['dead'])} shard(s)  PLE table: {ple_status}" + (f" ({ple_detail})" if ple_detail else ""))
    for n in (chk["missing"] + chk["extra"] + chk["problems"])[:10]:
        print(f"    {n}")
    ok = not (chk["missing"] or chk["extra"] or chk["problems"] or chk["dead"]) and ple_status in ("complete", "n/a")
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())