Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""BitNet 1.58-bit: the quantization allowlist, the export validators and the GGUF export.
Specs and validators for ``quantization='bitnet_1.58'`` and for
``soup export --format bitnet`` / ``--format tq1_0``, plus :func:`export_bitnet_gguf`,
the live llama.cpp conversion (v0.71.20 #134; it needs a built llama.cpp toolchain).
Training with ``onebitllms`` lives in :mod:`soup_cli.trainer.bitnet`, which gates its
setup on that package being installed.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
# Closed allowlist of BitNet-flavoured quant strings exposed to YAML.
BITNET_QUANT_FORMATS: frozenset[str] = frozenset({"bitnet_1.58"})
# Closed allowlist of BitNet export targets (the actual export formats).
BITNET_EXPORT_FORMATS: frozenset[str] = frozenset({"bitnet", "tq1_0"})
_BITNET_FAMILY_RE_PREFIXES: tuple[str, ...] = (
"bitnet", "falcon-e", "falcone", "1bitllm", "onebit",
)
@dataclass(frozen=True)
class BitNetSpec:
"""Metadata for the BitNet quant path. Frozen — immutable."""
name: str
description: str
bits: float
live_wired: bool
_BITNET_METADATA: Mapping[str, BitNetSpec] = MappingProxyType({
"bitnet_1.58": BitNetSpec(
name="bitnet_1.58",
description="BitNet 1.58-bit ternary weights (axolotl + onebitllms)",
bits=1.58,
live_wired=False,
),
})
def is_bitnet_quant(value: object) -> bool:
"""Return True iff ``value`` is a BitNet quant string."""
if isinstance(value, bool):
return False
if not isinstance(value, str):
return False
return value in BITNET_QUANT_FORMATS
def is_bitnet_export_format(value: object) -> bool:
"""Return True iff ``value`` is a BitNet export-format string."""
if isinstance(value, bool):
return False
if not isinstance(value, str):
return False
return value in BITNET_EXPORT_FORMATS
def get_bitnet_spec(name: str) -> BitNetSpec:
"""Return the frozen :class:`BitNetSpec` for ``name`` or raise."""
if not is_bitnet_quant(name):
supported = ", ".join(sorted(BITNET_QUANT_FORMATS))
raise ValueError(
f"BitNet quant {name!r} not supported. Supported: {supported}"
)
return _BITNET_METADATA[name]
def is_bitnet_model(model_name: object) -> bool:
"""Best-effort detect whether ``model_name`` references a BitNet family.
Checks every slash-delimited component (lowercased) against the
``_BITNET_FAMILY_RE_PREFIXES`` prefix list. This is intentionally more
permissive than v0.39.0 ``is_gemma4_model`` because BitNet families
typically live under namespaced orgs (``1bitllm/...``, ``OneBitLLM/...``)
rather than being identifiable by repo name alone.
"""
if isinstance(model_name, bool):
return False
if not isinstance(model_name, str):
return False
if not model_name or "\x00" in model_name:
return False
# Check each path component so an org name like "1bitllm/foo" matches
# while still rejecting unrelated substrings (e.g. an SFT model that
# happens to embed "bitnet" inside a description path).
for part in model_name.lower().split("/"):
if any(part.startswith(prefix) for prefix in _BITNET_FAMILY_RE_PREFIXES):
return True
return False
def validate_bitnet_export(format_name: object) -> str:
"""Validate a BitNet export-format string. Returns canonical form."""
if isinstance(format_name, bool):
raise TypeError(
f"bitnet export format must not be bool, got {format_name!r}"
)
if not isinstance(format_name, str):
raise TypeError(
f"bitnet export format must be str, "
f"got {type(format_name).__name__}"
)
if not format_name:
raise ValueError("bitnet export format must be non-empty")
if "\x00" in format_name:
raise ValueError(
"bitnet export format must not contain null bytes"
)
canonical = format_name.lower()
if canonical not in BITNET_EXPORT_FORMATS:
supported = ", ".join(sorted(BITNET_EXPORT_FORMATS))
raise ValueError(
f"bitnet export format {format_name!r} not supported. "
f"Supported: {supported}"
)
return canonical
# BitNet export-format → llama.cpp ternary quantize CLI arg. Both the
# friendly ``bitnet`` alias and the explicit ``tq1_0`` map to TQ1_0 (the
# llama.cpp 1.58-bit ternary GGUF type).
_BITNET_GGUF_QUANT_ARG: Mapping[str, str] = MappingProxyType({
"bitnet": "TQ1_0",
"tq1_0": "TQ1_0",
})
def export_bitnet_gguf(
*,
model_dir: str,
output_path: str,
export_format: str,
llama_cpp_dir: str,
) -> None:
"""Export a BitNet model as a TQ1_0 (1.58-bit ternary) GGUF (v0.71.20 #134).
Two-stage llama.cpp pipeline (reuses the v0.53.1 gguf machinery):
1. ``convert_hf_to_gguf.py`` → ``f16.gguf``
2. ``llama-quantize`` with the ``TQ1_0`` flavour → ``output_path``
No importance matrix is needed — ternary weights export directly. All
subprocess invocations use argv-list form (no shell). cwd containment +
symlink rejection mirror ``export_advanced_gguf``.
Requires a built llama.cpp toolchain; the convert/quantize binaries raise a
friendly ``FileNotFoundError`` naming the missing piece when absent.
"""
import os
import tempfile
from pathlib import Path
from soup_cli.utils.gguf_quant import (
_enforce_under_cwd_and_no_symlink,
_run_convert_to_f16,
_run_quantize_binary,
)
flavour = validate_bitnet_export(export_format)
quant_arg = _BITNET_GGUF_QUANT_ARG[flavour]
_enforce_under_cwd_and_no_symlink(model_dir, "model_dir")
_enforce_under_cwd_and_no_symlink(output_path, "output_path")
_enforce_under_cwd_and_no_symlink(llama_cpp_dir, "llama_cpp_dir")
if not os.path.isdir(model_dir):
raise FileNotFoundError(
f"model_dir not a directory: {os.path.basename(model_dir)!r}"
)
if not os.path.isdir(llama_cpp_dir):
raise FileNotFoundError(
f"llama_cpp_dir not a directory: {os.path.basename(llama_cpp_dir)!r}"
)
with tempfile.TemporaryDirectory(
prefix=".soup_bitnet_gguf_", dir=str(Path.cwd()),
) as staged:
f16_path = Path(staged) / "model.f16.gguf"
_run_convert_to_f16(llama_cpp_dir, model_dir, str(f16_path))
_run_quantize_binary(
llama_cpp_dir=llama_cpp_dir,
f16_path=str(f16_path),
output_path=output_path,
flavour=quant_arg,
imatrix_path=None,
)
if not os.path.isfile(output_path):
raise RuntimeError(
f"llama-quantize did not produce {os.path.basename(output_path)!r}"
)