Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
#!/usr/bin/env python3
"""Reconstruct STEP 6's layer-count control sweep.
The record reports exact streamed/resident gradients across the 48-to-64 layer
boundary while layers remain small, plus a wrong 48-layer NF4 control above the
per-layer-size bracket. The live mode creates those synthetic checkpoints.
"""
from __future__ import annotations
import argparse
import json
import math
import sys
from pathlib import Path
from typing import Any
class MeasurementInvalidError(RuntimeError):
"""The layer-count sweep is invalid."""
def layercount_verdict(samples: list[dict[str, Any]]) -> bool:
if len(samples) < 4:
return False
nf4_depths = set()
sweep_sizes: dict[str, set[float]] = {"nf4": set(), "bf16": set()}
bf16_sweep = False
positive_control = False
for sample in samples:
layers = sample.get("layers")
quant = sample.get("quant")
role = sample.get("role")
exact_counts = sample.get("exact_counts")
total = sample.get("total")
if quant not in {"nf4", "bf16"}:
raise MeasurementInvalidError("quant must be nf4 or bf16")
if role not in {"sweep", "control"}:
raise MeasurementInvalidError("role must be sweep or control")
if not isinstance(layers, int) or layers <= 0:
raise MeasurementInvalidError("layer counts must be positive integers")
if total != 4 * layers:
raise MeasurementInvalidError("total must equal four LoRA gradients per layer")
if not isinstance(exact_counts, list) or len(exact_counts) != 3:
raise MeasurementInvalidError("exact_counts must contain three backwards")
if any(not isinstance(exact, int) or exact < 0 or exact > total for exact in exact_counts):
raise MeasurementInvalidError("invalid exact gradient count")
value = float(sample.get("layer_mib", 0.0))
if not math.isfinite(value) or value <= 0:
raise MeasurementInvalidError("layer_mib must be finite and positive")
all_exact = all(exact == total for exact in exact_counts)
if role == "sweep":
if value > 163.8 or not all_exact:
return False
sweep_sizes[quant].add(round(value, 6))
if quant == "nf4":
nf4_depths.add(layers)
else:
bf16_sweep = True
else:
if quant != "nf4" or layers != 48 or value < 171.5:
return False
if not any(exact < total for exact in exact_counts[1:]):
return False
positive_control = True
spans_boundary = any(depth <= 48 for depth in nf4_depths) and any(
depth >= 64 for depth in nf4_depths
)
constant_sizes = all(len(sizes) <= 1 for sizes in sweep_sizes.values())
return spans_boundary and bf16_sweep and positive_control and constant_sizes
def measure_layer_sweep(*, layer_counts: list[int], measure) -> list[dict[str, Any]]:
"""Measure both small-layer sweeps and the above-bracket NF4 control."""
if len(set(layer_counts)) < 2:
raise MeasurementInvalidError("layer sweep must contain distinct counts")
rows = []
for layers in layer_counts:
for quant in ("nf4", "bf16"):
row = dict(
measure(
layers=layers,
hidden=64,
intermediate=128,
quant=quant,
role="sweep",
)
)
row.update(layers=layers, quant=quant, role="sweep")
rows.append(row)
control = dict(
measure(
layers=48,
hidden=5120,
intermediate=20480,
quant="nf4",
role="control",
)
)
control.update(layers=48, quant="nf4", role="control")
rows.append(control)
return rows
def _measure_checkpoint(weights: str, shards: str, quant: str) -> dict[str, Any]:
"""Run three streamed backwards against one resident reference."""
import bitexact
import torch
from mechanism_cost import (
MeasurementInvalidError as MechanismMeasurementInvalidError,
)
from mechanism_cost import (
build_streamed_arm,
gradient_snapshot,
make_input,
prepare_shards,
run_backward,
)
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import (
assert_canonical_parameters_intersect,
build_meta_skeleton,
build_streamed_model,
)
def restore() -> None:
return None
if quant == "nf4":
index, arch = prepare_shards(weights, shards)
streamed, runtime, restore = build_streamed_arm(weights, shards, index, arch, "control", 2)
else:
runtime_quant = "none"
probe = build_meta_skeleton(weights, dtype="bfloat16", quant=runtime_quant)
arch = bitexact.model_arch_name(probe)
del probe
index = shard_checkpoint(
weights,
shards,
dtype="bfloat16",
arch=arch,
quant=runtime_quant,
quant_suffixes=(),
double_quant=True,
quant_device="cuda",
)
streamed, runtime = build_streamed_model(
model_id=weights,
shard_dir=shards,
index=index,
lora_config=bitexact.lora_config(),
device="cuda",
dtype="bfloat16",
buffers=2,
pin=True,
seed=3,
quant=runtime_quant,
double_quant=True,
tier="ram",
)
reference = None
try:
reference = bitexact.load_resident_reference(weights, quant)
bitexact.make_non_vacuous_lora(streamed)
bitexact.copy_lora(streamed, reference)
assert_canonical_parameters_intersect(streamed, reference)
input_ids = make_input(reference, seq=128, batch=1)
reference_loss = run_backward(reference, input_ids)
try:
reference_grads = gradient_snapshot(reference)
except MechanismMeasurementInvalidError as exc:
raise MeasurementInvalidError(str(exc)) from exc
if (
not math.isfinite(reference_loss)
or not reference_grads
or any(not torch.isfinite(value).all() for value in reference_grads.values())
):
raise MeasurementInvalidError("resident measurement is empty or non-finite")
exact_counts = []
for _ in range(3):
loss = run_backward(streamed, input_ids)
try:
current = gradient_snapshot(streamed)
except MechanismMeasurementInvalidError as exc:
raise MeasurementInvalidError(str(exc)) from exc
if not math.isfinite(loss) or loss != reference_loss:
raise MeasurementInvalidError("streamed loss differs from resident")
if set(current) != set(reference_grads):
raise MeasurementInvalidError("gradient parameter sets differ")
if any(not torch.isfinite(value).all() for value in current.values()):
raise MeasurementInvalidError("streamed gradients are non-finite")
exact_counts.append(
sum(torch.equal(current[name], reference_grads[name]) for name in current)
)
store_bytes = getattr(getattr(runtime, "source", None), "nbytes", 0)
layers = getattr(index, "n_layers", 0)
if store_bytes <= 0 or layers <= 0:
raise MeasurementInvalidError("streaming store size is unavailable")
return {
"layer_mib": store_bytes / layers / (1024 * 1024),
"exact_counts": exact_counts,
"total": len(reference_grads),
}
finally:
if reference is not None:
del reference
try:
runtime.close()
finally:
restore()
del streamed
torch.cuda.empty_cache()
def run_live_sweep(work_dir: Path) -> list[dict[str, Any]]:
"""Create and measure the recorded depth sweep and positive control."""
import subprocess
script = Path(__file__).with_name("synth_checkpoint.py")
def measure(*, layers: int, hidden: int, intermediate: int, quant: str, role: str):
checkpoint = work_dir / f"llama-{role}-{layers}-{hidden}-{intermediate}"
shards = work_dir / f"shards-{role}-{layers}-{hidden}-{intermediate}-{quant}"
if not checkpoint.exists():
subprocess.run(
[
sys.executable,
str(script),
"--out",
str(checkpoint),
"--shape",
"qwen2.5-14b",
"--arch",
"llama",
"--layers",
str(layers),
"--hidden",
str(hidden),
"--intermediate",
str(intermediate),
"--heads",
"4" if hidden == 64 else "40",
"--kv-heads",
"2" if hidden == 64 else "10",
"--vocab",
"128" if hidden == 64 else "1024",
],
check=True,
)
shards.mkdir(parents=True, exist_ok=True)
return _measure_checkpoint(str(checkpoint), str(shards), quant)
return measure_layer_sweep(
layer_counts=[32, 48, 56, 60, 64, 80],
measure=measure,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Check the STEP 6 layer-count sweep.")
parser.add_argument("--records", type=Path, help="JSON array of measured sweep rows")
parser.add_argument("--measure", action="store_true", help="run the synthetic CUDA sweep")
parser.add_argument("--work-dir", type=Path, default=Path("./layercount-work"))
parser.add_argument("--self-test", action="store_true")
return parser.parse_args()
def run_self_test() -> int:
assert layercount_verdict(
[
{
"layers": 48,
"quant": "nf4",
"role": "sweep",
"layer_mib": 0.01,
"exact_counts": [192] * 3,
"total": 192,
},
{
"layers": 64,
"quant": "nf4",
"role": "sweep",
"layer_mib": 0.01,
"exact_counts": [256] * 3,
"total": 256,
},
{
"layers": 64,
"quant": "bf16",
"role": "sweep",
"layer_mib": 0.05,
"exact_counts": [256] * 3,
"total": 256,
},
{
"layers": 48,
"quant": "nf4",
"role": "control",
"layer_mib": 187.0,
"exact_counts": [192, 8, 8],
"total": 192,
},
]
)
print("PASS: layer-count verdict requires the sweep and positive control")
return 0
def main() -> int:
args = parse_args()
if args.self_test:
return run_self_test()
if args.measure:
import torch
if not torch.cuda.is_available():
print("SKIP: CUDA is required for the synthetic measurement")
return 0
try:
samples = run_live_sweep(args.work_dir.expanduser().resolve())
print(json.dumps(samples, indent=2, sort_keys=True))
if layercount_verdict(samples):
print("RESULT: layer-count control relationship reproduced")
return 0
print("ERROR: layer-count control relationship not reproduced")
return 1
except (MeasurementInvalidError, RuntimeError, ValueError) as exc:
print(f"ERROR: invalid measurement: {exc}")
return 3
if args.records is None:
print("ERROR: --records is required; no CUDA measurement was performed")
return 2
try:
samples = json.loads(args.records.read_text(encoding="utf-8"))
if not isinstance(samples, list):
raise MeasurementInvalidError("records must be a JSON array")
if not layercount_verdict(samples):
print("ERROR: layer-count control relationship not reproduced")
return 1
print("RESULT: layer-count control relationship reproduced")
return 0
except MeasurementInvalidError as exc:
print(f"ERROR: invalid measurement: {exc}")
return 3
except (OSError, json.JSONDecodeError, TypeError) as exc:
print(f"ERROR: invalid records: {exc}")
return 2
if __name__ == "__main__":
sys.exit(main())