FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""QuantMethod: one (kind, layer kind) pair bound to one kernel object; the selector and the post-load hook."""
from __future__ import annotations
from abc import ABC
from typing import Any, ClassVar, Sequence
from freetoken.utils import init_logger
from .registry import LayerKind
from .scheme import QuantKind
logger = init_logger(__name__)
class KernelSelectionError(RuntimeError):
pass
def select_kernel(candidates: Sequence[type], requested: str, cfg: Any):
"""Pick one kernel from an ordered table: the requested name if usable, else the first usable one that is worth it."""
if not candidates:
raise KernelSelectionError("empty kernel table")
names = [c.name for c in candidates]
if requested != "auto" and requested in names:
kernel = candidates[names.index(requested)]()
reason = kernel.unusable_reason(cfg)
if reason:
raise KernelSelectionError(f"kernel {requested!r} was requested but cannot run here: {reason}")
return kernel
skipped: list[str] = []
fallback = None
for cls in candidates:
kernel = cls()
reason = kernel.unusable_reason(cfg)
if reason:
skipped.append(f"{kernel.name}: {reason}")
continue
if kernel.worth_it(cfg):
if skipped:
logger.info("kernel %s selected; skipped %s", kernel.name, "; ".join(skipped))
return kernel
if fallback is None:
fallback = kernel
if fallback is not None:
return fallback
raise KernelSelectionError("no usable kernel in table; " + "; ".join(skipped))
class QuantMethod(ABC):
kind: ClassVar[QuantKind]
layer_kind: ClassVar[LayerKind]
candidates: ClassVar[tuple[type, ...]] = ()
def __init__(self, cfg: Any, requested: str = "auto"):
self.cfg = cfg
self.requested = requested
self.kernel = select_kernel(self.candidates, requested, cfg)
@property
def scheme(self):
return self.cfg.scheme
def finalize_quant(root: Any) -> int:
"""Call ``quant_method.finalize(layer)`` on every layer under ``root``; returns the count."""
from freetoken.layers.base import iter_ops
count = 0
for op in iter_ops(root):
method = getattr(op, "quant_method", None)
if method is not None:
method.finalize(op)
count += 1
return count